# 1 "<built-in>"
# 1 "beam/beam_emu.c"
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifdef HAVE_CONFIG_H
#if 0 /* expanded by -frewrite-includes */
#  include "config.h"
#endif /* expanded by -frewrite-includes */
# 22 "beam/beam_emu.c"
# 1 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h" 1
/* armv6-portbld-freebsd11.0/config.h.  Generated by configure.  */
/* config.h.in.  Generated from configure.in by autoheader.  */


#define GHBN_R_SOLARIS  2
#define GHBN_R_AIX      3
#define GHBN_R_GLIBC    4


/* Assumed cache-line size (in bytes) */
#define ASSUMED_CACHE_LINE_SIZE 64

/* Define the brk() argument type. */
#define BRK_ARG_TYPE const void *

/* Define the brk() return type. */
#define BRK_RET_TYPE int

/* Modern style mcontext_t in MacOSX */
/* #undef DARWIN_MODERN_MCONTEXT */

/* Define if you need to include rpc/types.h to get INADDR_LOOPBACK defined */
/* #undef DEF_INADDR_LOOPBACK_IN_RPC_TYPES_H */

/* Define if you need to include winsock2.h to get INADDR_LOOPBACK defined */
/* #undef DEF_INADDR_LOOPBACK_IN_WINSOCK2_H */

/* Define if you want to disable child waiter thread */
/* #undef DISABLE_CHILD_WAITER_THREAD */

/* Define if you want to disable vfork. */
/* #undef DISABLE_VFORK */

/* Define to 1 if your processor stores the words in a double in middle-endian
   format (like some ARMs). */
/* #undef DOUBLE_MIDDLE_ENDIAN */

/* Define if you want to enable child waiter thread */
/* #undef ENABLE_CHILD_WAITER_THREAD */

/* Dirty scheduler support */
/* #undef ERL_DRV_DIRTY_SCHEDULER_SUPPORT */

/* Dirty scheduler support */
/* #undef ERL_NIF_DIRTY_SCHEDULER_SUPPORT */

/* Define if sbrk()/brk() wrappers can track malloc()s core memory use */
/* #undef ERTS_BRK_WRAPPERS_CAN_TRACK_MALLOC */

/* Define to override the default number of write_concurrency locks */
/* #undef ERTS_DB_HASH_LOCK_CNT */

/* Define if the emulator supports dirty schedulers */
/* #undef ERTS_DIRTY_SCHEDULERS */

/* The only reason ERTS_EMU_CMDLINE_FLAGS exists is to force modification of
   config.h when the emulator command line flags are modified by configure */
#define ERTS_EMU_CMDLINE_FLAGS " -O -pipe -mfloat-abi=softfp -fno-strict-aliasing -I/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0   -g  -I/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0    -DUSE_THREADS -D_THREAD_SAFE -D_REENTRANT -DPOSIX_THREADS  -Werror=return-type  -Wall -Wstrict-prototypes -Wmissing-prototypes -Wdeclaration-after-statement"

/* Define if you have kernel poll and want to use it */
#define ERTS_ENABLE_KERNEL_POLL 1

/* Define if OS monotonic clock is corrected */
/* #undef ERTS_HAVE_CORRECTED_OS_MONOTONIC_TIME */

/* Define if you have a low resolution OS monotonic clock */
/* #undef ERTS_HAVE_LOW_RESOLUTION_OS_MONOTONIC_LOW */

/* Define if the smp emulator is built */
#define ERTS_HAVE_SMP_EMU 1

/* Define if dlopen() needs to be called before first call to dlerror() */
/* #undef ERTS_NEED_DLOPEN_BEFORE_DLERROR */

/* Save compile time? */
#define ERTS_SAVED_COMPILE_TIME 1

/* Define if poll() should be used instead of select() */
#define ERTS_USE_POLL 1

/* Define if __after_morecore_hook can track malloc()s core memory use. */
/* #undef ERTS___AFTER_MORECORE_HOOK_CAN_TRACK_MALLOC */

/* Define if bigendian */
/* #undef ETHR_BIGENDIAN */

/* Define if gcc wont let you clobber ebx with cmpxchg8b and position
   independent code */
/* #undef ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX */

/* Define if you get a register shortage with cmpxchg8b and position
   independent code */
/* #undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE */

/* Define if you want to disable native ethread implementations */
/* #undef ETHR_DISABLE_NATIVE_IMPLS */

/* Define if you want to force usage of pthread rwlocks */
/* #undef ETHR_FORCE_PTHREAD_RWLOCK */

/* Define if you use a gcc that supports the double word cmpxchg instruction
   */
/* #undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */

/* Define if you use a gcc that supports -msse2 and understand sse2 specific
   asm statements */
/* #undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT */

/* Define if you have a clock_gettime() with a monotonic clock */
#define ETHR_HAVE_CLOCK_GETTIME_MONOTONIC 1

/* Define if you have all ethread defines */
#define ETHR_HAVE_ETHREAD_DEFINES 1

/* Define as a boolean indicating whether you have a gcc compatible compiler
   capable of generating the ARM DMB instruction, and are compiling for an ARM
   processor with ARM DMB instruction support, or not */
#define ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION 0

/* Define as a boolean indicating whether you have a gcc __atomic builtins or
   not */
#define ETHR_HAVE_GCC___ATOMIC_BUILTINS 1

/* Define if you have a monotonic gethrtime() */
/* #undef ETHR_HAVE_GETHRTIME */

/* Define if you have libatomic_ops atomic operations */
/* #undef ETHR_HAVE_LIBATOMIC_OPS */

/* Define if you have a linux futex implementation. */
/* #undef ETHR_HAVE_LINUX_FUTEX */

/* Define if you have a mach clock_get_time() with a monotonic clock */
/* #undef ETHR_HAVE_MACH_CLOCK_GET_TIME */

/* Define if the pthread.h header file is in pthread/mit directory. */
/* #undef ETHR_HAVE_MIT_PTHREAD_H */

/* Define if you have the pthread_attr_setguardsize function. */
#define ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE 1

/* Define if pthread_cond_timedwait() can be used with a monotonic clock */
#define ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC 1

/* Define if you have ibm style pthread_getname_np */
/* #undef ETHR_HAVE_PTHREAD_GETNAME_NP_2 */

/* Define if you have linux style pthread_getname_np */
/* #undef ETHR_HAVE_PTHREAD_GETNAME_NP_3 */

/* Define if you have the <pthread.h> header file. */
#define ETHR_HAVE_PTHREAD_H 1

/* Define if you have the pthread_rwlockattr_setkind_np() function. */
/* #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP */

/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
   attribute. */
/* #undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP */

/* Define if you have darwin style pthread_setname_np */
/* #undef ETHR_HAVE_PTHREAD_SETNAME_NP_1 */

/* Define if you have linux style pthread_setname_np */
/* #undef ETHR_HAVE_PTHREAD_SETNAME_NP_2 */

/* Define if you have bsd style pthread_set_name_np */
/* #undef ETHR_HAVE_PTHREAD_SET_NAME_NP_2 */

/* Define if you have the pthread_spin_lock function. */
#define ETHR_HAVE_PTHREAD_SPIN_LOCK 1

/* Define if you have the pthread_yield() function. */
#define ETHR_HAVE_PTHREAD_YIELD 1

/* Define if you have the <sched.h> header file. */
#define ETHR_HAVE_SCHED_H 1

/* Define if you have the sched_yield() function. */
#define ETHR_HAVE_SCHED_YIELD 1

/* Define if you have the <sys/time.h> header file. */
#define ETHR_HAVE_SYS_TIME_H 1

/* Define if you have _InterlockedAnd() */
/* #undef ETHR_HAVE__INTERLOCKEDAND */

/* Define if you have _InterlockedAnd64() */
/* #undef ETHR_HAVE__INTERLOCKEDAND64 */

/* Define if you have _InterlockedCompareExchange() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE */

/* Define if you have _InterlockedCompareExchange128() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128 */

/* Define if you have _InterlockedCompareExchange64() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 */

/* Define if you have _InterlockedCompareExchange64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ */

/* Define if you have _InterlockedCompareExchange64_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL */

/* Define if you have _InterlockedCompareExchange_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ */

/* Define if you have _InterlockedCompareExchange_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL */

/* Define if you have _InterlockedDecrement() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT */

/* Define if you have _InterlockedDecrement64() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT64 */

/* Define if you have _InterlockedDecrement64_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL */

/* Define if you have _InterlockedDecrement_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL */

/* Define if you have _InterlockedExchange() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGE */

/* Define if you have _InterlockedExchange64() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 */

/* Define if you have _InterlockedExchangeAdd() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD */

/* Define if you have _InterlockedExchangeAdd64() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 */

/* Define if you have _InterlockedExchangeAdd64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ */

/* Define if you have _InterlockedExchangeAdd_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ */

/* Define if you have _InterlockedIncrement() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT */

/* Define if you have _InterlockedIncrement64() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT64 */

/* Define if you have _InterlockedIncrement64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ */

/* Define if you have _InterlockedIncrement_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ */

/* Define if you have _InterlockedOr() */
/* #undef ETHR_HAVE__INTERLOCKEDOR */

/* Define if you have _InterlockedOr64() */
/* #undef ETHR_HAVE__INTERLOCKEDOR64 */

/* Define as a bitmask corresponding to the word sizes that
   __atomic_add_fetch() can handle on your system */
#define ETHR_HAVE___atomic_add_fetch 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_compare_exchange_n() can handle on your system */
#define ETHR_HAVE___atomic_compare_exchange_n 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_fetch_and() can handle on your system */
#define ETHR_HAVE___atomic_fetch_and 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_fetch_or() can handle on your system */
#define ETHR_HAVE___atomic_fetch_or 12

/* Define as a bitmask corresponding to the word sizes that __atomic_load_n()
   can handle on your system */
#define ETHR_HAVE___atomic_load_n 12

/* Define as a bitmask corresponding to the word sizes that __atomic_store_n()
   can handle on your system */
#define ETHR_HAVE___atomic_store_n 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_add_and_fetch() can handle on your system */
#define ETHR_HAVE___sync_add_and_fetch 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_fetch_and_and() can handle on your system */
#define ETHR_HAVE___sync_fetch_and_and 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_fetch_and_or() can handle on your system */
#define ETHR_HAVE___sync_fetch_and_or 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_synchronize() can handle on your system */
#define ETHR_HAVE___sync_synchronize ~0

/* Define as a bitmask corresponding to the word sizes that
   __sync_val_compare_and_swap() can handle on your system */
#define ETHR_HAVE___sync_val_compare_and_swap 12

/* Define if you want to modify the default stack size */
/* #undef ETHR_MODIFIED_DEFAULT_STACK_SIZE */

/* Define to the monotonic clock id to use */
#define ETHR_MONOTONIC_CLOCK_ID CLOCK_MONOTONIC

/* Define if you need the <nptl/pthread.h> header file. */
/* #undef ETHR_NEED_NPTL_PTHREAD_H */

/* Define if you have OSE style threads */
/* #undef ETHR_OSE_THREADS */

/* Define if you prefer gcc native ethread implementations */
/* #undef ETHR_PREFER_GCC_NATIVE_IMPLS */

/* Define if you prefer libatomic_ops native ethread implementations */
/* #undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS */

/* Define if you have pthreads */
#define ETHR_PTHREADS 1

/* Define if pthread_yield() returns an int. */
/* #undef ETHR_PTHREAD_YIELD_RET_INT */

/* Define if sched_yield() returns an int. */
#define ETHR_SCHED_YIELD_RET_INT 1

/* Define to the size of AO_t if libatomic_ops is used */
/* #undef ETHR_SIZEOF_AO_T */

/* Define to the size of int */
#define ETHR_SIZEOF_INT 4

/* Define to the size of long */
#define ETHR_SIZEOF_LONG 4

/* Define to the size of long long */
#define ETHR_SIZEOF_LONG_LONG 8

/* Define to the size of pointers */
#define ETHR_SIZEOF_PTR 4

/* Define to the size of __int128_t */
#define ETHR_SIZEOF___INT128_T 0

/* Define to the size of __int64 */
#define ETHR_SIZEOF___INT64 0

/* Define if you want to enable check for native ethread implementations */
#define ETHR_SMP_REQUIRE_NATIVE_IMPLS 1

/* Define if only run in Sparc PSO, or TSO mode */
/* #undef ETHR_SPARC_PSO */

/* Define if run in Sparc RMO, PSO, or TSO mode */
/* #undef ETHR_SPARC_RMO */

/* Define if only run in Sparc TSO mode */
/* #undef ETHR_SPARC_TSO */

/* Define if you can safely include both <sys/time.h> and <time.h>. */
#define ETHR_TIME_WITH_SYS_TIME 1

/* Define as a boolean indicating whether you trust gcc's __atomic_* builtins
   memory barrier implementations, or not */
#define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS 0

/* Define if you have win32 threads */
/* #undef ETHR_WIN32_THREADS */

/* Define if x86/x86_64 out of order instructions should be synchronized */
/* #undef ETHR_X86_OUT_OF_ORDER */

/* Define if building a halfword-heap 64bit emulator */
/* #undef HALFWORD_HEAP_EMULATOR */

/* Define to 1 if you have the <arpa/nameser.h> header file. */
#define HAVE_ARPA_NAMESER_H 1

/* Define to 1 if you have the `brk' function. */
#define HAVE_BRK 1

/* Define to 1 if you have the `clock_getres' function. */
#define HAVE_CLOCK_GETRES 1

/* define if clock_gettime() works for getting process time */
/* #undef HAVE_CLOCK_GETTIME_CPU_TIME */

/* Define if you have clock_gettime(CLOCK_MONOTONIC_RAW, _) */
/* #undef HAVE_CLOCK_GETTIME_MONOTONIC_RAW */

/* Define to 1 if you have the `clock_get_attributes' function. */
/* #undef HAVE_CLOCK_GET_ATTRIBUTES */

/* Define to 1 if you have the `closefrom' function. */
#define HAVE_CLOSEFROM 1

/* Define if you have a decl of fread that conflicts with int fread */
#define HAVE_CONFLICTING_FREAD_DECLARATION 1

/* Define if you have a putenv() that stores a copy of the key-value pair */
/* #undef HAVE_COPYING_PUTENV */

/* Define if you have cpuset_getaffinity/cpuset_setaffinity */
#define HAVE_CPUSET_xETAFFINITY 1

/* Define to 1 if you have the declaration of `getrlimit', and to 0 if you
   don't. */
#define HAVE_DECL_GETRLIMIT 1

/* Define to 1 if you have the declaration of `IN6ADDR_ANY_INIT', and to 0 if
   you don't. */
#define HAVE_DECL_IN6ADDR_ANY_INIT 1

/* Define to 1 if you have the declaration of `IN6ADDR_LOOPBACK_INIT', and to
   0 if you don't. */
#define HAVE_DECL_IN6ADDR_LOOPBACK_INIT 1

/* Define to 1 if you have the declaration of `IPV6_V6ONLY', and to 0 if you
   don't. */
#define HAVE_DECL_IPV6_V6ONLY 1

/* Define to 1 if you have the declaration of `posix2time', and to 0 if you
   don't. */
#define HAVE_DECL_POSIX2TIME 0

/* Define to 1 if you have the declaration of `RLIMIT_STACK', and to 0 if you
   don't. */
#define HAVE_DECL_RLIMIT_STACK 1

/* Define to 1 if you have the declaration of `SCTPS_BOUND', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_BOUND 0

/* Define to 1 if you have the declaration of `SCTPS_COOKIE_ECHOED', and to 0
   if you don't. */
#define HAVE_DECL_SCTPS_COOKIE_ECHOED 0

/* Define to 1 if you have the declaration of `SCTPS_COOKIE_WAIT', and to 0 if
   you don't. */
#define HAVE_DECL_SCTPS_COOKIE_WAIT 0

/* Define to 1 if you have the declaration of `SCTPS_ESTABLISHED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTPS_ESTABLISHED 0

/* Define to 1 if you have the declaration of `SCTPS_IDLE', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_IDLE 0

/* Define to 1 if you have the declaration of `SCTPS_LISTEN', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_LISTEN 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_ACK_SENT', and
   to 0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_ACK_SENT 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_PENDING', and to
   0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_PENDING 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_RECEIVED', and
   to 0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_RECEIVED 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_SENT', and to 0
   if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_SENT 0

/* Define to 1 if you have the declaration of `SCTP_ABORT', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_ABORT 1

/* Define to 1 if you have the declaration of `SCTP_ADDR_CONFIRMED', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_ADDR_CONFIRMED 1

/* Define to 1 if you have the declaration of `SCTP_ADDR_OVER', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_ADDR_OVER 1

/* Define to 1 if you have the declaration of `SCTP_BOUND', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_BOUND 1

/* Define to 1 if you have the declaration of `SCTP_CLOSED', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_CLOSED 1

/* Define to 1 if you have the declaration of `SCTP_COOKIE_ECHOED', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_COOKIE_ECHOED 1

/* Define to 1 if you have the declaration of `SCTP_COOKIE_WAIT', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_COOKIE_WAIT 1

/* Define to 1 if you have the declaration of `SCTP_DELAYED_ACK_TIME', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_DELAYED_ACK_TIME 0

/* Define to 1 if you have the declaration of `SCTP_EMPTY', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_EMPTY 0

/* Define to 1 if you have the declaration of `SCTP_EOF', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_EOF 1

/* Define to 1 if you have the declaration of `SCTP_ESTABLISHED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_ESTABLISHED 1

/* Define to 1 if you have the declaration of `SCTP_LISTEN', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_LISTEN 1

/* Define to 1 if you have the declaration of `SCTP_SENDALL', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_SENDALL 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_ACK_SENT', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_ACK_SENT 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_PENDING', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_PENDING 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_RECEIVED', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_RECEIVED 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_SENT', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_SENT 1

/* Define to 1 if you have the declaration of `SCTP_UNCONFIRMED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_UNCONFIRMED 1

/* Define to 1 if you have the declaration of `SCTP_UNORDERED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_UNORDERED 1

/* Define to 1 if you have the declaration of `setrlimit', and to 0 if you
   don't. */
#define HAVE_DECL_SETRLIMIT 1

/* Define to 1 if you have the declaration of `time2posix', and to 0 if you
   don't. */
#define HAVE_DECL_TIME2POSIX 0

/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
   */
#define HAVE_DIRENT_H 1

/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1

/* Define to 1 if you have the `dlopen' function. */
#define HAVE_DLOPEN 1

/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
/* #undef HAVE_DOPRNT */

/* Define if you have the 'end' symbol */
#define HAVE_END_SYMBOL 1

/* Define if you have a working fallocate() */
/* #undef HAVE_FALLOCATE */

/* Define to 1 if you have the <fcntl.h> header file. */
#define HAVE_FCNTL_H 1

/* Define to 1 if you have the `fdatasync' function. */
/* #undef HAVE_FDATASYNC */

/* Define to 1 if you have the `finite' function. */
#define HAVE_FINITE 1

/* Define to 1 if you have the `flockfile' function. */
#define HAVE_FLOCKFILE 1

/* Define to 1 if you have the `fork' function. */
#define HAVE_FORK 1

/* Define to 1 if you have the `fpsetmask' function. */
#define HAVE_FPSETMASK 1

/* Define to 1 if you have the `fstat' function. */
#define HAVE_FSTAT 1

/* Define if you have fwrite_unlocked */
/* #undef HAVE_FWRITE_UNLOCKED */

/* Define to 1 if you have a good `getaddrinfo' function. */
#define HAVE_GETADDRINFO 1

/* Define to 1 if you have the `gethostbyname2' function. */
#define HAVE_GETHOSTBYNAME2 1

/* Define to flavour of gethostbyname_r */
/* #undef HAVE_GETHOSTBYNAME_R */

/* Define to 1 if you have the `gethrtime' function. */
/* #undef HAVE_GETHRTIME */

/* define if gethrvtime() works and uses ioctl() to /proc/self */
/* #undef HAVE_GETHRVTIME_PROCFS_IOCTL */

/* Define to 1 if you have the `getifaddrs' function. */
#define HAVE_GETIFADDRS 1

/* Define to 1 if you have the `getipnodebyaddr' function. */
#define HAVE_GETIPNODEBYADDR 1

/* Define to 1 if you have the `getipnodebyname' function. */
#define HAVE_GETIPNODEBYNAME 1

/* Define to 1 if you have a good `getnameinfo' function. */
#define HAVE_GETNAMEINFO 1

/* Define to 1 if you have the `gettimeofday' function. */
#define HAVE_GETTIMEOFDAY 1

/* Define to 1 if you have the `gmtime_r' function. */
#define HAVE_GMTIME_R 1

/* Define to 1 if you have the <ieeefp.h> header file. */
#define HAVE_IEEEFP_H 1

/* Define to 1 if you have the `ieee_handler' function. */
/* #undef HAVE_IEEE_HANDLER */

/* Define to 1 if you have the <ifaddrs.h> header file. */
#define HAVE_IFADDRS_H 1

/* Define if ipv6 is present */
#define HAVE_IN6 1

/* Define to 1 if you have the variable in6addr_any declared. */
#define HAVE_IN6ADDR_ANY 1

/* Define to 1 if you have the variable in6addr_loopback declared. */
#define HAVE_IN6ADDR_LOOPBACK 1

/* Define to 1 if you have the `inet_pton' function. */
#define HAVE_INET_PTON 1

/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1

/* Early linux used in_addr6 instead of in6_addr, define if you have this */
/* #undef HAVE_IN_ADDR6_STRUCT */

/* Define to 1 if you have the `isfinite' function. */
/* #undef HAVE_ISFINITE */

/* Define to 1 if you have the `isinf' function. */
#define HAVE_ISINF 1

/* Define to 1 if you have the `isnan' function. */
#define HAVE_ISNAN 1

/* Define if you have kstat */
/* #undef HAVE_KSTAT */

/* Define to 1 if you have the <langinfo.h> header file. */
#define HAVE_LANGINFO_H 1

/* Define to 1 if you have the `dl' library (-ldl). */
/* #undef HAVE_LIBDL */

/* Define to 1 if you have the `dlpi' library (-ldlpi). */
/* #undef HAVE_LIBDLPI */

/* Define to 1 if you have the <libdlpi.h> header file. */
/* #undef HAVE_LIBDLPI_H */

/* Define to 1 if you have the `inet' library (-linet). */
/* #undef HAVE_LIBINET */

/* Define to 1 if you have the `m' library (-lm). */
#define HAVE_LIBM 1

/* Define to 1 if you have the `util' library (-lutil). */
#define HAVE_LIBUTIL 1

/* Define to 1 if you have the `z' library (-lz). */
#define HAVE_LIBZ 1

/* Define to 1 if you have the <limits.h> header file. */
#define HAVE_LIMITS_H 1

/* Define to 1 if you have the <linux/falloc.h> header file. */
/* #undef HAVE_LINUX_FALLOC_H */

/* Define to 1 if you have the `localtime_r' function. */
#define HAVE_LOCALTIME_R 1

/* Define to 1 if you have the `log2' function. */
#define HAVE_LOG2 1

/* Define to 1 if you have the <malloc.h> header file. */
/* #undef HAVE_MALLOC_H */

/* Define to 1 if you have the `mallopt' function. */
/* #undef HAVE_MALLOPT */

/* Define to 1 if you have the `memcpy' function. */
#define HAVE_MEMCPY 1

/* Define to 1 if you have the `memmove' function. */
#define HAVE_MEMMOVE 1

/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1

/* Define if the pthread.h header file is in pthread/mit directory. */
/* #undef HAVE_MIT_PTHREAD_H */

/* Define to 1 if you have the `mlockall' function. */
#define HAVE_MLOCKALL 1

/* Define to 1 if you have the `mmap' function. */
#define HAVE_MMAP 1

/* Define if you have a monotonic erts_os_hrtime() implementation */
/* #undef HAVE_MONOTONIC_ERTS_SYS_HRTIME */

/* Define to 1 if you have the `mremap' function. */
/* #undef HAVE_MREMAP */

/* Define if setsockopt() accepts multicast options */
#define HAVE_MULTICAST_SUPPORT 1

/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
/* #undef HAVE_NDIR_H */

/* Define to 1 if you have the <netpacket/packet.h> header file. */
/* #undef HAVE_NETPACKET_PACKET_H */

/* Define to 1 if you have the <net/errno.h> header file. */
/* #undef HAVE_NET_ERRNO_H */

/* Define to 1 if you have the <net/if_dl.h> header file. */
#define HAVE_NET_IF_DL_H 1

/* Define to 1 if you have the `nl_langinfo' function. */
#define HAVE_NL_LANGINFO 1

/* Define if you don't have a definition of INADDR_LOOPBACK */
/* #undef HAVE_NO_INADDR_LOOPBACK */

/* Define to 1 if you have the `openpty' function. */
#define HAVE_OPENPTY 1

/* Define if you have the "ose_spi/ose_spi.h" header file. */
/* #undef HAVE_OSE_SPI_H */

/* Define to 1 if you have the `poll' function. */
#define HAVE_POLL 1

/* Define to 1 if you have the <poll.h> header file. */
#define HAVE_POLL_H 1

/* Define to 1 if you have the `posix2time' function. */
#define HAVE_POSIX2TIME 1

/* Define to 1 if you have the `posix_fadvise' function. */
#define HAVE_POSIX_FADVISE 1

/* Define if you have a working posix_fallocate() */
#define HAVE_POSIX_FALLOCATE 

/* Define to 1 if you have the `posix_memalign' function. */
#define HAVE_POSIX_MEMALIGN 1

/* Define to 1 if you have the `ppoll' function. */
#define HAVE_PPOLL 1

/* Define to 1 if you have the `pread' function. */
#define HAVE_PREAD 1

/* Define if you have processor_bind functionality */
/* #undef HAVE_PROCESSOR_BIND */

/* Define if you have pset functionality */
/* #undef HAVE_PSET */

/* Define if you have the <pthread.h> header file. */
#define HAVE_PTHREAD_H 1

/* Define to 1 if you have the <pty.h> header file. */
/* #undef HAVE_PTY_H */

/* Define if you have putc_unlocked */
#define HAVE_PUTC_UNLOCKED 1

/* Define to 1 if you have the `pwrite' function. */
#define HAVE_PWRITE 1

/* Define to 1 if you have the `res_gethostbyname' function. */
/* #undef HAVE_RES_GETHOSTBYNAME */

/* Define to 1 if you have the `sbrk' function. */
#define HAVE_SBRK 1

/* Define to 1 if you have the <sched.h> header file. */
#define HAVE_SCHED_H 1

/* Define if you have sched_getaffinity/sched_setaffinity */
/* #undef HAVE_SCHED_xETAFFINITY */

/* Define to 1 if you have the `sctp_bindx' function. */
#define HAVE_SCTP_BINDX 1

/* Define to 1 if you have the `sctp_freeladdrs' function. */
#define HAVE_SCTP_FREELADDRS 1

/* Define to 1 if you have the `sctp_freepaddrs' function. */
#define HAVE_SCTP_FREEPADDRS 1

/* Define to 1 if you have the `sctp_getladdrs' function. */
#define HAVE_SCTP_GETLADDRS 1

/* Define to 1 if you have the `sctp_getpaddrs' function. */
#define HAVE_SCTP_GETPADDRS 1

/* Define to 1 if you have the <netinet/sctp.h> header file */
#define HAVE_SCTP_H 1

/* Define to 1 if you have the `sctp_peeloff' function. */
#define HAVE_SCTP_PEELOFF 1

/* Define to 1 if you have the <sdkddkver.h> header file. */
/* #undef HAVE_SDKDDKVER_H */

/* Define to 1 if you have the `sendfile' function. */
#define HAVE_SENDFILE 1

/* Define to 1 if you have the `sendfilev' function. */
/* #undef HAVE_SENDFILEV */

/* Define to 1 if you have the `setlocale' function. */
#define HAVE_SETLOCALE 1

/* Define to 1 if you have the `setns' function. */
/* #undef HAVE_SETNS */

/* Define to 1 if you have the <setns.h> header file. */
/* #undef HAVE_SETNS_H */

/* Define to 1 if you have the `setsid' function. */
#define HAVE_SETSID 1

/* Define if we have socklen_t */
#define HAVE_SOCKLEN_T 1

/* define if you have the Solaris/ultrasparc /dev/perfmon interface */
/* #undef HAVE_SOLARIS_SPARC_PERFMON */

/* Define if you have SO_BSDCOMPAT flag on sockets */
/* #undef HAVE_SO_BSDCOMPAT */

/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1

/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1

/* Define to 1 if you have the `strerror' function. */
#define HAVE_STRERROR 1

/* Define to 1 if you have the `strerror_r' function. */
#define HAVE_STRERROR_R 1

/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1

/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1

/* Define to 1 if you have the `strlcat' function. */
#define HAVE_STRLCAT 1

/* Define to 1 if you have the `strlcpy' function. */
#define HAVE_STRLCPY 1

/* Define to 1 if you have the `strncasecmp' function. */
#define HAVE_STRNCASECMP 1

/* Define to 1 if `ifr_enaddr' is member of `struct ifreq'. */
/* #undef HAVE_STRUCT_IFREQ_IFR_ENADDR */

/* Define to 1 if `ifr_hwaddr' is member of `struct ifreq'. */
/* #undef HAVE_STRUCT_IFREQ_IFR_HWADDR */

/* Define to 1 if `spp_flags' is member of `struct sctp_paddrparams'. */
#define HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS 1

/* Define to 1 if `spp_pathmtu' is member of `struct sctp_paddrparams'. */
#define HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_PATHMTU 1

/* Define to 1 if `spp_sackdelay' is member of `struct sctp_paddrparams'. */
/* #undef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY */

/* Define to 1 if `sre_data' is member of `struct sctp_remote_error'. */
#define HAVE_STRUCT_SCTP_REMOTE_ERROR_SRE_DATA 1

/* Define to 1 if `ssf_data' is member of `struct sctp_send_failed'. */
#define HAVE_STRUCT_SCTP_SEND_FAILED_SSF_DATA 1

/* Define to 1 if you have the <syslog.h> header file. */
#define HAVE_SYSLOG_H 1

/* Define if you have systemd daemon */
/* #undef HAVE_SYSTEMD_DAEMON */

/* Define to 1 if you have the <systemd/sd-daemon.h> header file. */
/* #undef HAVE_SYSTEMD_SD_DAEMON_H */

/* Define if you have <sys/devpoll.h> header file. */
/* #undef HAVE_SYS_DEVPOLL_H */

/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
   */
/* #undef HAVE_SYS_DIR_H */

/* Define if you have the <sys/epoll.h> header file. */
/* #undef HAVE_SYS_EPOLL_H */

/* Define if you have <sys/event.h> header file. */
#define HAVE_SYS_EVENT_H 1

/* Define to 1 if you have the <sys/ioctl.h> header file. */
#define HAVE_SYS_IOCTL_H 1

/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
   */
/* #undef HAVE_SYS_NDIR_H */

/* Define to 1 if you have the <sys/resource.h> header file */
#define HAVE_SYS_RESOURCE_H 1

/* Define to 1 if you have the <sys/sdt.h> header file. */
/* #undef HAVE_SYS_SDT_H */

/* Define to 1 if you have the <sys/socketio.h> header file. */
/* #undef HAVE_SYS_SOCKETIO_H */

/* Define to 1 if you have the <sys/socket.h> header file. */
#define HAVE_SYS_SOCKET_H 1

/* Define to 1 if you have the <sys/sockio.h> header file. */
#define HAVE_SYS_SOCKIO_H 1

/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1

/* Define to 1 if you have the <sys/stropts.h> header file. */
/* #undef HAVE_SYS_STROPTS_H */

/* Define to 1 if you have the <sys/sysctl.h> header file. */
#define HAVE_SYS_SYSCTL_H 1

/* Define to 1 if you have the <sys/timerfd.h> header file. */
/* #undef HAVE_SYS_TIMERFD_H */

/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1

/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1

/* Define to 1 if you have the <sys/uio.h> header file. */
#define HAVE_SYS_UIO_H 1

/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */
#define HAVE_SYS_WAIT_H 1

/* Define if termcap functions exists */
#define HAVE_TERMCAP 1

/* Define to 1 if you have the `time2posix' function. */
#define HAVE_TIME2POSIX 1

/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1

/* Define to 1 if you have the <util.h> header file. */
/* #undef HAVE_UTIL_H */

/* Define to 1 if you have the <utmp.h> header file. */
/* #undef HAVE_UTMP_H */

/* Define to 1 if you have the `vfork' function. */
#define HAVE_VFORK 1

/* Define to 1 if you have the <vfork.h> header file. */
/* #undef HAVE_VFORK_H */

/* Define to 1 if you have the `vprintf' function. */
#define HAVE_VPRINTF 1

/* Define to 1 if you have a `wcwidth' function. */
#define HAVE_WCWIDTH 1

/* Define to 1 if you have the <windows.h> header file. */
/* #undef HAVE_WINDOWS_H */

/* Define to 1 if you have the <winsock2.h> header file. */
/* #undef HAVE_WINSOCK2_H */

/* Define to 1 if `fork' works. */
#define HAVE_WORKING_FORK 1

/* Define if you have a working posix_openpt implementation */
#define HAVE_WORKING_POSIX_OPENPT 1

/* Define to 1 if `vfork' works. */
#define HAVE_WORKING_VFORK 1

/* Define to 1 if you have the `writev' function. */
#define HAVE_WRITEV 1

/* Define to 1 if you have the <ws2tcpip.h> header file. */
/* #undef HAVE_WS2TCPIP_H */

/* Define to 1 if you have the `_brk' function. */
#define HAVE__BRK 1

/* Define if you have the '_end' symbol */
#define HAVE__END_SYMBOL 1

/* Define to 1 if you have the `_sbrk' function. */
#define HAVE__SBRK 1

/* Define to 1 if you have the `__brk' function. */
/* #undef HAVE___BRK */

/* Define to 1 if you have the `__sbrk' function. */
/* #undef HAVE___SBRK */

/* Define to enable HiPE */
#define HIPE 1

/* Define to monotonic clock id to use */
#define HRTIME_CLOCK_ID CLOCK_UPTIME_PRECISE

/* Define as a string of monotonic clock id to use */
#define HRTIME_CLOCK_ID_STR "CLOCK_UPTIME_PRECISE"

/* define if h_errno is declared (in some way) in a system header file */
#define H_ERRNO_DECLARED 1

/* Define to monotonic clock id to use */
#define MONOTONIC_CLOCK_ID CLOCK_UPTIME

/* Define as a string of monotonic clock id to use */
#define MONOTONIC_CLOCK_ID_STR "CLOCK_UPTIME"

/* Define if netdb.h needs struct sockaddr_in ans in.h CAN be included before
   */
/* #undef NETDB_H_NEEDS_IN_H */

/* Define if floating points exceptions are non-existing/not reliable */
#define NO_FPE_SIGNALS 

/* Defined if no found C compiler can handle jump tables */
/* #undef NO_JUMP_TABLE */

/* Define if you dont have salen */
/* #undef NO_SA_LEN */

/* Define if you want to implement erts_os_monotonic_time() using
   clock_gettime() */
#define OS_MONOTONIC_TIME_USING_CLOCK_GETTIME 1

/* Define if you want to implement erts_os_monotonic_time() using gethrtime()
   */
/* #undef OS_MONOTONIC_TIME_USING_GETHRTIME */

/* Define if you want to implement erts_os_monotonic_time() using mach
   clock_get_time() */
/* #undef OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME */

/* Define if you want to implement erts_os_monotonic_time() using times() */
/* #undef OS_MONOTONIC_TIME_USING_TIMES */

/* Define if you want to implement erts_os_system_time() using gettimeofday()
   */
/* #undef OS_SYSTEM_TIME_GETTIMEOFDAY */

/* Define if you want to implement erts_os_system_time() using clock_gettime()
   */
#define OS_SYSTEM_TIME_USING_CLOCK_GETTIME 1

/* Define if you want to implement erts_os_system_time() using mach
   clock_get_time() */
/* #undef OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME */

/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT ""

/* Define to the full name of this package. */
#define PACKAGE_NAME ""

/* Define to the full name and version of this package. */
#define PACKAGE_STRING ""

/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME ""

/* Define to the version of this package. */
#define PACKAGE_VERSION ""

/* Define if you wish to redefine FD_SETSIZE to be able to select on more fd
   */
/* #undef REDEFINE_FD_SETSIZE */

/* Define as the return type of signal handlers (`int' or `void'). */
#define RETSIGTYPE void

/* Define the sbrk() argument type. */
#define SBRK_ARG_TYPE intptr_t

/* Define the sbrk() return type. */
#define SBRK_RET_TYPE void *

/* The size of a `AO_t', as computed by sizeof. */
/* #undef SIZEOF_AO_T */

/* The size of a `char', as computed by sizeof. */
#define SIZEOF_CHAR 1

/* The size of a `int', as computed by sizeof. */
#define SIZEOF_INT 4

/* The size of a `long', as computed by sizeof. */
#define SIZEOF_LONG 4

/* The size of a `long long', as computed by sizeof. */
#define SIZEOF_LONG_LONG 8

/* The size of a `off_t', as computed by sizeof. */
#define SIZEOF_OFF_T 8

/* The size of a `short', as computed by sizeof. */
#define SIZEOF_SHORT 2

/* The size of a `size_t', as computed by sizeof. */
#define SIZEOF_SIZE_T 4

/* The size of a `time_t', as computed by sizeof. */
#define SIZEOF_TIME_T 8

/* The size of a `void *', as computed by sizeof. */
#define SIZEOF_VOID_P 4

/* The size of a `__int128_t', as computed by sizeof. */
#define SIZEOF___INT128_T 0

/* The size of a `__int64', as computed by sizeof. */
#define SIZEOF___INT64 0

/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1

/* define if the variable sys_errlist is declared in a system header file */
#define SYS_ERRLIST_DECLARED 

/* Define if you want to implement erts_os_hrtime() using clock_gettime() */
/* #undef SYS_HRTIME_USING_CLOCK_GETTIME */

/* Define if you want to implement erts_os_hrtime() using gethrtime() */
/* #undef SYS_HRTIME_USING_GETHRTIME */

/* Define if you want to implement erts_os_hrtime() using mach
   clock_get_time() */
/* #undef SYS_HRTIME_USING_MACH_CLOCK_GET_TIME */

/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#define TIME_WITH_SYS_TIME 1

/* Define to 1 if your <sys/time.h> declares `struct tm'. */
/* #undef TM_IN_SYS_TIME */

/* Define if you want to use dtrace for dynamic tracing */
/* #undef USE_DTRACE */

/* Define if you want to use dynamic tracing */
/* #undef USE_DYNAMIC_TRACE */

/* Define if you have matherr() function and struct exception type */
/* #undef USE_MATHERR */

/* Define if select() should be used instead of poll() */
/* #undef USE_SELECT */

/* Define if you want to use systemtap for dynamic tracing */
/* #undef USE_SYSTEMTAP */

/* Define to enable VM dynamic trace probes */
/* #undef USE_VM_PROBES */

/* Define to wall clock id to use */
#define WALL_CLOCK_ID CLOCK_REALTIME

/* Define as a string of wall clock id to use */
#define WALL_CLOCK_ID_STR "CLOCK_REALTIME"

/* Define if windows.h includes winsock2.h */
/* #undef WINDOWS_H_INCLUDES_WINSOCK2_H */

/* Define to 1 if your processor stores words with the most significant byte
   first (like Motorola and SPARC, unlike Intel and VAX). */
/* #undef WORDS_BIGENDIAN */

/* Define to empty if `const' does not conform to ANSI C. */
/* #undef const */

/* Define to `long' if <sys/types.h> does not define. */
/* #undef off_t */

/* Define to `int' if <sys/types.h> does not define. */
/* #undef pid_t */

/* Define to `unsigned' if <sys/types.h> does not define. */
/* #undef size_t */

/* Define as `fork' if `vfork' does not work. */
/* #undef vfork */


/* Redefine in6_addr. XXX this should be moved to the files where it's used? */
#ifdef HAVE_IN_ADDR6_STRUCT
#define in6_addr in_addr6
#endif
# 1245 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

/* Define a reasonable default for INADDR_LOOPBACK */
/* XXX this should be moved to the files where it's used? */
#ifdef HAVE_NO_INADDR_LOOPBACK
#define INADDR_LOOPBACK (u_long)0x7F000001
#endif
# 1251 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#ifdef REDEFINE_FD_SETSIZE
#define FD_SETSIZE 1024
#endif
# 1255 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
#define HAVE_GETHRVTIME
#endif
# 1259 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#if !defined(HAVE_ISFINITE) && !defined(HAVE_FINITE)
# if defined(HAVE_ISINF) && defined(HAVE_ISNAN)
#  define USE_ISINF_ISNAN
# endif
# 1264 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"
#endif
# 1265 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#if defined(DEBUG) && defined(USE_THREADS) && !defined(ERTS_ENABLE_LOCK_CHECK)
#define ERTS_ENABLE_LOCK_CHECK 1
#endif
# 1269 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

# 23 "beam/beam_emu.c" 2
#endif
# 24 "beam/beam_emu.c"

#if 0 /* expanded by -frewrite-includes */
#include <stddef.h> /* offsetof() */
#endif /* expanded by -frewrite-includes */
# 25 "beam/beam_emu.c"
# 1 "/usr/include/stddef.h" 1 3 4
/*-
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)stddef.h	8.1 (Berkeley) 6/2/93
 *
 * $FreeBSD: head/include/stddef.h 270277 2014-08-21 15:10:10Z kan $
 */

#ifndef _STDDEF_H_
#define _STDDEF_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/stddef.h" 3 4
# 1 "/usr/include/sys/cdefs.h" 1 3 4
/*-
 * Copyright (c) 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * Berkeley Software Design, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)cdefs.h	8.8 (Berkeley) 1/9/95
 * $FreeBSD: head/sys/sys/cdefs.h 295041 2016-01-29 15:12:31Z br $
 */

#ifndef	_SYS_CDEFS_H_
#define	_SYS_CDEFS_H_

/*
 * Testing against Clang-specific extensions.
 */
#ifndef	__has_attribute
#define	__has_attribute(x)	0
#endif
# 45 "/usr/include/sys/cdefs.h" 3 4
#ifndef	__has_extension
#define	__has_extension		__has_feature
#endif
# 48 "/usr/include/sys/cdefs.h" 3 4
#ifndef	__has_feature
#define	__has_feature(x)	0
#endif
# 51 "/usr/include/sys/cdefs.h" 3 4
#ifndef	__has_include
#define	__has_include(x)	0
#endif
# 54 "/usr/include/sys/cdefs.h" 3 4
#ifndef	__has_builtin
#define	__has_builtin(x)	0
#endif
# 57 "/usr/include/sys/cdefs.h" 3 4

#if defined(__cplusplus)
#define	__BEGIN_DECLS	extern "C" {
#define	__END_DECLS	}
#else
# 62 "/usr/include/sys/cdefs.h" 3 4
#define	__BEGIN_DECLS
#define	__END_DECLS
#endif
# 65 "/usr/include/sys/cdefs.h" 3 4

/*
 * This code has been put in place to help reduce the addition of
 * compiler specific defines in FreeBSD code.  It helps to aid in
 * having a compiler-agnostic source tree.
 */

#if defined(__GNUC__) || defined(__INTEL_COMPILER)

#if __GNUC__ >= 3 || defined(__INTEL_COMPILER)
#define	__GNUCLIKE_ASM 3
#define	__GNUCLIKE_MATH_BUILTIN_CONSTANTS
#else
# 78 "/usr/include/sys/cdefs.h" 3 4
#define	__GNUCLIKE_ASM 2
#endif
# 80 "/usr/include/sys/cdefs.h" 3 4
#define	__GNUCLIKE___TYPEOF 1
#define	__GNUCLIKE___OFFSETOF 1
#define	__GNUCLIKE___SECTION 1

#ifndef __INTEL_COMPILER
#define	__GNUCLIKE_CTOR_SECTION_HANDLING 1
#endif
# 87 "/usr/include/sys/cdefs.h" 3 4

#define	__GNUCLIKE_BUILTIN_CONSTANT_P 1
#if defined(__INTEL_COMPILER) && defined(__cplusplus) && \
   __INTEL_COMPILER < 800
#undef __GNUCLIKE_BUILTIN_CONSTANT_P
#endif
# 93 "/usr/include/sys/cdefs.h" 3 4

#if (__GNUC_MINOR__ > 95 || __GNUC__ >= 3)
#define	__GNUCLIKE_BUILTIN_VARARGS 1
#define	__GNUCLIKE_BUILTIN_STDARG 1
#define	__GNUCLIKE_BUILTIN_VAALIST 1
#endif
# 99 "/usr/include/sys/cdefs.h" 3 4

#if defined(__GNUC__)
#define	__GNUC_VA_LIST_COMPATIBILITY 1
#endif
# 103 "/usr/include/sys/cdefs.h" 3 4

/*
 * Compiler memory barriers, specific to gcc and clang.
 */
#if defined(__GNUC__)
#define	__compiler_membar()	__asm __volatile(" " : : : "memory")
#endif
# 110 "/usr/include/sys/cdefs.h" 3 4

#ifndef __INTEL_COMPILER
#define	__GNUCLIKE_BUILTIN_NEXT_ARG 1
#define	__GNUCLIKE_MATH_BUILTIN_RELOPS
#endif
# 115 "/usr/include/sys/cdefs.h" 3 4

#define	__GNUCLIKE_BUILTIN_MEMCPY 1

/* XXX: if __GNUC__ >= 2: not tested everywhere originally, where replaced */
#define	__CC_SUPPORTS_INLINE 1
#define	__CC_SUPPORTS___INLINE 1
#define	__CC_SUPPORTS___INLINE__ 1

#define	__CC_SUPPORTS___FUNC__ 1
#define	__CC_SUPPORTS_WARNING 1

#define	__CC_SUPPORTS_VARADIC_XXX 1 /* see varargs.h */

#define	__CC_SUPPORTS_DYNAMIC_ARRAY_INIT 1

#endif /* __GNUC__ || __INTEL_COMPILER */
# 131 "/usr/include/sys/cdefs.h" 3 4

/*
 * Macro to test if we're using a specific version of gcc or later.
 */
#if defined(__GNUC__) && !defined(__INTEL_COMPILER)
#define	__GNUC_PREREQ__(ma, mi)	\
	(__GNUC__ > (ma) || __GNUC__ == (ma) && __GNUC_MINOR__ >= (mi))
#else
# 139 "/usr/include/sys/cdefs.h" 3 4
#define	__GNUC_PREREQ__(ma, mi)	0
#endif
# 141 "/usr/include/sys/cdefs.h" 3 4

/*
 * The __CONCAT macro is used to concatenate parts of symbol names, e.g.
 * with "#define OLD(foo) __CONCAT(old,foo)", OLD(foo) produces oldfoo.
 * The __CONCAT macro is a bit tricky to use if it must work in non-ANSI
 * mode -- there must be no spaces between its arguments, and for nested
 * __CONCAT's, all the __CONCAT's must be at the left.  __CONCAT can also
 * concatenate double-quoted strings produced by the __STRING macro, but
 * this only works with ANSI C.
 *
 * __XSTRING is like __STRING, but it expands any macros in its argument
 * first.  It is only available with ANSI C.
 */
#if defined(__STDC__) || defined(__cplusplus)
#define	__P(protos)	protos		/* full-blown ANSI C */
#define	__CONCAT1(x,y)	x ## y
#define	__CONCAT(x,y)	__CONCAT1(x,y)
#define	__STRING(x)	#x		/* stringify without expanding x */
#define	__XSTRING(x)	__STRING(x)	/* expand x, then stringify */

#define	__const		const		/* define reserved names to standard */
#define	__signed	signed
#define	__volatile	volatile
#if defined(__cplusplus)
#define	__inline	inline		/* convert to C++ keyword */
#else
# 167 "/usr/include/sys/cdefs.h" 3 4
#if !(defined(__CC_SUPPORTS___INLINE))
#define	__inline			/* delete GCC keyword */
#endif /* ! __CC_SUPPORTS___INLINE */
# 170 "/usr/include/sys/cdefs.h" 3 4
#endif /* !__cplusplus */
# 171 "/usr/include/sys/cdefs.h" 3 4

#else	/* !(__STDC__ || __cplusplus) */
# 173 "/usr/include/sys/cdefs.h" 3 4
#define	__P(protos)	()		/* traditional C preprocessor */
#define	__CONCAT(x,y)	x/**/y
#define	__STRING(x)	"x"

#if !defined(__CC_SUPPORTS___INLINE)
#define	__const				/* delete pseudo-ANSI C keywords */
#define	__inline
#define	__signed
#define	__volatile
/*
 * In non-ANSI C environments, new programs will want ANSI-only C keywords
 * deleted from the program and old programs will want them left alone.
 * When using a compiler other than gcc, programs using the ANSI C keywords
 * const, inline etc. as normal identifiers should define -DNO_ANSI_KEYWORDS.
 * When using "gcc -traditional", we assume that this is the intent; if
 * __GNUC__ is defined but __STDC__ is not, we leave the new keywords alone.
 */
#ifndef	NO_ANSI_KEYWORDS
#define	const				/* delete ANSI C keywords */
#define	inline
#define	signed
#define	volatile
#endif	/* !NO_ANSI_KEYWORDS */
# 196 "/usr/include/sys/cdefs.h" 3 4
#endif	/* !__CC_SUPPORTS___INLINE */
# 197 "/usr/include/sys/cdefs.h" 3 4
#endif	/* !(__STDC__ || __cplusplus) */
# 198 "/usr/include/sys/cdefs.h" 3 4

/*
 * Compiler-dependent macros to help declare dead (non-returning) and
 * pure (no side effects) functions, and unused variables.  They are
 * null except for versions of gcc that are known to support the features
 * properly (old versions of gcc-2 supported the dead and pure features
 * in a different (wrong) way).  If we do not provide an implementation
 * for a given compiler, let the compile fail if it is told to use
 * a feature that we cannot live without.
 */
#ifdef lint
#define	__dead2
#define	__pure2
#define	__unused
#define	__packed
#define	__aligned(x)
#define	__alloc_align(x)
#define	__alloc_size(x)
#define	__section(x)
#define	__weak_symbol
#else
# 219 "/usr/include/sys/cdefs.h" 3 4
#define	__weak_symbol	__attribute__((__weak__))
#if !__GNUC_PREREQ__(2, 5) && !defined(__INTEL_COMPILER)
#define	__dead2
#define	__pure2
#define	__unused
#endif
# 225 "/usr/include/sys/cdefs.h" 3 4
#if __GNUC__ == 2 && __GNUC_MINOR__ >= 5 && __GNUC_MINOR__ < 7 && !defined(__INTEL_COMPILER)
#define	__dead2		__attribute__((__noreturn__))
#define	__pure2		__attribute__((__const__))
#define	__unused
/* XXX Find out what to do for __packed, __aligned and __section */
#endif
# 231 "/usr/include/sys/cdefs.h" 3 4
#if __GNUC_PREREQ__(2, 7) || defined(__INTEL_COMPILER)
#define	__dead2		__attribute__((__noreturn__))
#define	__pure2		__attribute__((__const__))
#define	__unused	__attribute__((__unused__))
#define	__used		__attribute__((__used__))
#define	__packed	__attribute__((__packed__))
#define	__aligned(x)	__attribute__((__aligned__(x)))
#define	__section(x)	__attribute__((__section__(x)))
#endif
# 240 "/usr/include/sys/cdefs.h" 3 4
#if __GNUC_PREREQ__(4, 3) || __has_attribute(__alloc_size__)
#define	__alloc_size(x)	__attribute__((__alloc_size__(x)))
#else
# 243 "/usr/include/sys/cdefs.h" 3 4
#define	__alloc_size(x)
#endif
# 245 "/usr/include/sys/cdefs.h" 3 4
#if __GNUC_PREREQ__(4, 9) || __has_attribute(__alloc_align__)
#define	__alloc_align(x)	__attribute__((__alloc_align__(x)))
#else
# 248 "/usr/include/sys/cdefs.h" 3 4
#define	__alloc_align(x)
#endif
# 250 "/usr/include/sys/cdefs.h" 3 4
#endif /* lint */
# 251 "/usr/include/sys/cdefs.h" 3 4

#if !__GNUC_PREREQ__(2, 95)
#define	__alignof(x)	__offsetof(struct { char __a; x __b; }, __b)
#endif
# 255 "/usr/include/sys/cdefs.h" 3 4

/*
 * Keywords added in C11.
 */

#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112L || defined(lint)

#if !__has_extension(c_alignas)
#if (defined(__cplusplus) && __cplusplus >= 201103L) || \
    __has_extension(cxx_alignas)
#define	_Alignas(x)		alignas(x)
#else
# 267 "/usr/include/sys/cdefs.h" 3 4
/* XXX: Only emulates _Alignas(constant-expression); not _Alignas(type-name). */
#define	_Alignas(x)		__aligned(x)
#endif
# 270 "/usr/include/sys/cdefs.h" 3 4
#endif
# 271 "/usr/include/sys/cdefs.h" 3 4

#if defined(__cplusplus) && __cplusplus >= 201103L
#define	_Alignof(x)		alignof(x)
#else
# 275 "/usr/include/sys/cdefs.h" 3 4
#define	_Alignof(x)		__alignof(x)
#endif
# 277 "/usr/include/sys/cdefs.h" 3 4

#if !__has_extension(c_atomic) && !__has_extension(cxx_atomic)
/*
 * No native support for _Atomic(). Place object in structure to prevent
 * most forms of direct non-atomic access.
 */
#define	_Atomic(T)		struct { T volatile __val; }
#endif
# 285 "/usr/include/sys/cdefs.h" 3 4

#if defined(__cplusplus) && __cplusplus >= 201103L
#define	_Noreturn		[[noreturn]]
#else
# 289 "/usr/include/sys/cdefs.h" 3 4
#define	_Noreturn		__dead2
#endif
# 291 "/usr/include/sys/cdefs.h" 3 4

#if !__has_extension(c_static_assert)
#if (defined(__cplusplus) && __cplusplus >= 201103L) || \
    __has_extension(cxx_static_assert)
#define	_Static_assert(x, y)	static_assert(x, y)
#elif __GNUC_PREREQ__(4,6)
# 297 "/usr/include/sys/cdefs.h" 3 4
/* Nothing, gcc 4.6 and higher has _Static_assert built-in */
#elif defined(__COUNTER__)
# 299 "/usr/include/sys/cdefs.h" 3 4
#define	_Static_assert(x, y)	__Static_assert(x, __COUNTER__)
#define	__Static_assert(x, y)	___Static_assert(x, y)
#define	___Static_assert(x, y)	typedef char __assert_ ## y[(x) ? 1 : -1] \
				__unused
#else
# 304 "/usr/include/sys/cdefs.h" 3 4
#define	_Static_assert(x, y)	struct __hack
#endif
# 306 "/usr/include/sys/cdefs.h" 3 4
#endif
# 307 "/usr/include/sys/cdefs.h" 3 4

#if !__has_extension(c_thread_local)
/*
 * XXX: Some compilers (Clang 3.3, GCC 4.7) falsely announce C++11 mode
 * without actually supporting the thread_local keyword. Don't check for
 * the presence of C++11 when defining _Thread_local.
 */
#if /* (defined(__cplusplus) && __cplusplus >= 201103L) || */ \
    __has_extension(cxx_thread_local)
#define	_Thread_local		thread_local
#else
# 318 "/usr/include/sys/cdefs.h" 3 4
#define	_Thread_local		__thread
#endif
# 320 "/usr/include/sys/cdefs.h" 3 4
#endif
# 321 "/usr/include/sys/cdefs.h" 3 4

#endif /* __STDC_VERSION__ || __STDC_VERSION__ < 201112L */
# 323 "/usr/include/sys/cdefs.h" 3 4

/*
 * Emulation of C11 _Generic().  Unlike the previously defined C11
 * keywords, it is not possible to implement this using exactly the same
 * syntax.  Therefore implement something similar under the name
 * __generic().  Unlike _Generic(), this macro can only distinguish
 * between a single type, so it requires nested invocations to
 * distinguish multiple cases.
 */

#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || \
    __has_extension(c_generic_selections)
#define	__generic(expr, t, yes, no)					\
	_Generic(expr, t: yes, default: no)
#elif __GNUC_PREREQ__(3, 1) && !defined(__cplusplus)
# 338 "/usr/include/sys/cdefs.h" 3 4
#define	__generic(expr, t, yes, no)					\
	__builtin_choose_expr(						\
	    __builtin_types_compatible_p(__typeof(expr), t), yes, no)
#endif
# 342 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(2, 96)
#define	__malloc_like	__attribute__((__malloc__))
#define	__pure		__attribute__((__pure__))
#else
# 347 "/usr/include/sys/cdefs.h" 3 4
#define	__malloc_like
#define	__pure
#endif
# 350 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(3, 1) || (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 800)
#define	__always_inline	__attribute__((__always_inline__))
#else
# 354 "/usr/include/sys/cdefs.h" 3 4
#define	__always_inline
#endif
# 356 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(3, 1)
#define	__noinline	__attribute__ ((__noinline__))
#else
# 360 "/usr/include/sys/cdefs.h" 3 4
#define	__noinline
#endif
# 362 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(3, 3)
#define	__nonnull(x)	__attribute__((__nonnull__(x)))
#define	__nonnull_all	__attribute__((__nonnull__))
#else
# 367 "/usr/include/sys/cdefs.h" 3 4
#define	__nonnull(x)
#define	__nonnull_all
#endif
# 370 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(3, 4)
#define	__fastcall	__attribute__((__fastcall__))
#define	__result_use_check	__attribute__((__warn_unused_result__))
#else
# 375 "/usr/include/sys/cdefs.h" 3 4
#define	__fastcall
#define	__result_use_check
#endif
# 378 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(4, 1)
#define	__returns_twice	__attribute__((__returns_twice__))
#else
# 382 "/usr/include/sys/cdefs.h" 3 4
#define	__returns_twice
#endif
# 384 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(4, 6) || __has_builtin(__builtin_unreachable)
#define	__unreachable()	__builtin_unreachable()
#else
# 388 "/usr/include/sys/cdefs.h" 3 4
#define	__unreachable()	((void)0)
#endif
# 390 "/usr/include/sys/cdefs.h" 3 4

/* XXX: should use `#if __STDC_VERSION__ < 199901'. */
#if !__GNUC_PREREQ__(2, 7) && !defined(__INTEL_COMPILER)
#define	__func__	NULL
#endif
# 395 "/usr/include/sys/cdefs.h" 3 4

#if (defined(__INTEL_COMPILER) || (defined(__GNUC__) && __GNUC__ >= 2)) && !defined(__STRICT_ANSI__) || __STDC_VERSION__ >= 199901
#define	__LONG_LONG_SUPPORTED
#endif
# 399 "/usr/include/sys/cdefs.h" 3 4

/* C++11 exposes a load of C99 stuff */
#if defined(__cplusplus) && __cplusplus >= 201103L
#define	__LONG_LONG_SUPPORTED
#ifndef	__STDC_LIMIT_MACROS
#define	__STDC_LIMIT_MACROS
#endif
# 406 "/usr/include/sys/cdefs.h" 3 4
#ifndef	__STDC_CONSTANT_MACROS
#define	__STDC_CONSTANT_MACROS
#endif
# 409 "/usr/include/sys/cdefs.h" 3 4
#endif
# 410 "/usr/include/sys/cdefs.h" 3 4

/*
 * GCC 2.95 provides `__restrict' as an extension to C90 to support the
 * C99-specific `restrict' type qualifier.  We happen to use `__restrict' as
 * a way to define the `restrict' type qualifier without disturbing older
 * software that is unaware of C99 keywords.
 */
#if !(__GNUC__ == 2 && __GNUC_MINOR__ == 95)
#if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901 || defined(lint)
#define	__restrict
#else
# 421 "/usr/include/sys/cdefs.h" 3 4
#define	__restrict	restrict
#endif
# 423 "/usr/include/sys/cdefs.h" 3 4
#endif
# 424 "/usr/include/sys/cdefs.h" 3 4

/*
 * GNU C version 2.96 adds explicit branch prediction so that
 * the CPU back-end can hint the processor and also so that
 * code blocks can be reordered such that the predicted path
 * sees a more linear flow, thus improving cache behavior, etc.
 *
 * The following two macros provide us with a way to utilize this
 * compiler feature.  Use __predict_true() if you expect the expression
 * to evaluate to true, and __predict_false() if you expect the
 * expression to evaluate to false.
 *
 * A few notes about usage:
 *
 *	* Generally, __predict_false() error condition checks (unless
 *	  you have some _strong_ reason to do otherwise, in which case
 *	  document it), and/or __predict_true() `no-error' condition
 *	  checks, assuming you want to optimize for the no-error case.
 *
 *	* Other than that, if you don't know the likelihood of a test
 *	  succeeding from empirical or other `hard' evidence, don't
 *	  make predictions.
 *
 *	* These are meant to be used in places that are run `a lot'.
 *	  It is wasteful to make predictions in code that is run
 *	  seldomly (e.g. at subsystem initialization time) as the
 *	  basic block reordering that this affects can often generate
 *	  larger code.
 */
#if __GNUC_PREREQ__(2, 96)
#define	__predict_true(exp)     __builtin_expect((exp), 1)
#define	__predict_false(exp)    __builtin_expect((exp), 0)
#else
# 457 "/usr/include/sys/cdefs.h" 3 4
#define	__predict_true(exp)     (exp)
#define	__predict_false(exp)    (exp)
#endif
# 460 "/usr/include/sys/cdefs.h" 3 4

#if __GNUC_PREREQ__(4, 0)
#define	__null_sentinel	__attribute__((__sentinel__))
#define	__exported	__attribute__((__visibility__("default")))
#define	__hidden	__attribute__((__visibility__("hidden")))
#else
# 466 "/usr/include/sys/cdefs.h" 3 4
#define	__null_sentinel
#define	__exported
#define	__hidden
#endif
# 470 "/usr/include/sys/cdefs.h" 3 4

/*
 * We define this here since <stddef.h>, <sys/queue.h>, and <sys/types.h>
 * require it.
 */
#if __GNUC_PREREQ__(4, 1)
#define	__offsetof(type, field)	 __builtin_offsetof(type, field)
#else
# 478 "/usr/include/sys/cdefs.h" 3 4
#ifndef __cplusplus
#define	__offsetof(type, field) \
	((__size_t)(__uintptr_t)((const volatile void *)&((type *)0)->field))
#else
# 482 "/usr/include/sys/cdefs.h" 3 4
#define	__offsetof(type, field)					\
  (__offsetof__ (reinterpret_cast <__size_t>			\
                 (&reinterpret_cast <const volatile char &>	\
                  (static_cast<type *> (0)->field))))
#endif
# 487 "/usr/include/sys/cdefs.h" 3 4
#endif
# 488 "/usr/include/sys/cdefs.h" 3 4
#define	__rangeof(type, start, end) \
	(__offsetof(type, end) - __offsetof(type, start))

/*
 * Given the pointer x to the member m of the struct s, return
 * a pointer to the containing structure.  When using GCC, we first
 * assign pointer x to a local variable, to check that its type is
 * compatible with member m.
 */
#if __GNUC_PREREQ__(3, 1)
#define	__containerof(x, s, m) ({					\
	const volatile __typeof(((s *)0)->m) *__x = (x);		\
	__DEQUALIFY(s *, (const volatile char *)__x - __offsetof(s, m));\
})
#else
# 503 "/usr/include/sys/cdefs.h" 3 4
#define	__containerof(x, s, m)						\
	__DEQUALIFY(s *, (const volatile char *)(x) - __offsetof(s, m))
#endif
# 506 "/usr/include/sys/cdefs.h" 3 4

/*
 * Compiler-dependent macros to declare that functions take printf-like
 * or scanf-like arguments.  They are null except for versions of gcc
 * that are known to support the features properly (old versions of gcc-2
 * didn't permit keeping the keywords out of the application namespace).
 */
#if !__GNUC_PREREQ__(2, 7) && !defined(__INTEL_COMPILER)
#define	__printflike(fmtarg, firstvararg)
#define	__scanflike(fmtarg, firstvararg)
#define	__format_arg(fmtarg)
#define	__strfmonlike(fmtarg, firstvararg)
#define	__strftimelike(fmtarg, firstvararg)
#else
# 520 "/usr/include/sys/cdefs.h" 3 4
#define	__printflike(fmtarg, firstvararg) \
	    __attribute__((__format__ (__printf__, fmtarg, firstvararg)))
#define	__scanflike(fmtarg, firstvararg) \
	    __attribute__((__format__ (__scanf__, fmtarg, firstvararg)))
#define	__format_arg(fmtarg)	__attribute__((__format_arg__ (fmtarg)))
#define	__strfmonlike(fmtarg, firstvararg) \
	    __attribute__((__format__ (__strfmon__, fmtarg, firstvararg)))
#define	__strftimelike(fmtarg, firstvararg) \
	    __attribute__((__format__ (__strftime__, fmtarg, firstvararg)))
#endif
# 530 "/usr/include/sys/cdefs.h" 3 4

/*
 * FORTIFY_SOURCE, and perhaps other compiler-specific features, require
 * the use of non-standard inlining.  In general we should try to avoid
 * using these but GCC-compatible compilers tend to support the extensions
 * well enough to use them in limited cases.
 */ 
#if defined(__GNUC_GNU_INLINE__) || defined(__GNUC_STDC_INLINE__)
#if __GNUC_PREREQ__(4, 3) || __has_attribute(__artificial__)
#define	__gnu_inline	__attribute__((__gnu_inline__, __artificial__))
#else
# 541 "/usr/include/sys/cdefs.h" 3 4
#define	__gnu_inline	__attribute__((__gnu_inline__))
#endif /* artificial */
# 543 "/usr/include/sys/cdefs.h" 3 4
#else
# 544 "/usr/include/sys/cdefs.h" 3 4
#define	__gnu_inline
#endif
# 546 "/usr/include/sys/cdefs.h" 3 4

/* Compiler-dependent macros that rely on FreeBSD-specific extensions. */
#if defined(__FreeBSD_cc_version) && __FreeBSD_cc_version >= 300001 && \
    defined(__GNUC__) && !defined(__INTEL_COMPILER)
#define	__printf0like(fmtarg, firstvararg) \
	    __attribute__((__format__ (__printf0__, fmtarg, firstvararg)))
#else
# 553 "/usr/include/sys/cdefs.h" 3 4
#define	__printf0like(fmtarg, firstvararg)
#endif
# 555 "/usr/include/sys/cdefs.h" 3 4

#if defined(__GNUC__) || defined(__INTEL_COMPILER)
#ifndef __INTEL_COMPILER
#define	__strong_reference(sym,aliassym)	\
	extern __typeof (sym) aliassym __attribute__ ((__alias__ (#sym)))
#endif
# 561 "/usr/include/sys/cdefs.h" 3 4
#ifdef __STDC__
#define	__weak_reference(sym,alias)	\
	__asm__(".weak " #alias);	\
	__asm__(".equ "  #alias ", " #sym)
#define	__warn_references(sym,msg)	\
	__asm__(".section .gnu.warning." #sym);	\
	__asm__(".asciz \"" msg "\"");	\
	__asm__(".previous")
#define	__sym_compat(sym,impl,verid)	\
	__asm__(".symver " #impl ", " #sym "@" #verid)
#define	__sym_default(sym,impl,verid)	\
	__asm__(".symver " #impl ", " #sym "@@" #verid)
#else
# 574 "/usr/include/sys/cdefs.h" 3 4
#define	__weak_reference(sym,alias)	\
	__asm__(".weak alias");		\
	__asm__(".equ alias, sym")
#define	__warn_references(sym,msg)	\
	__asm__(".section .gnu.warning.sym"); \
	__asm__(".asciz \"msg\"");	\
	__asm__(".previous")
#define	__sym_compat(sym,impl,verid)	\
	__asm__(".symver impl, sym@verid")
#define	__sym_default(impl,sym,verid)	\
	__asm__(".symver impl, sym@@verid")
#endif	/* __STDC__ */
# 586 "/usr/include/sys/cdefs.h" 3 4
#endif	/* __GNUC__ || __INTEL_COMPILER */
# 587 "/usr/include/sys/cdefs.h" 3 4

#define	__GLOBL1(sym)	__asm__(".globl " #sym)
#define	__GLOBL(sym)	__GLOBL1(sym)

#if defined(__GNUC__) || defined(__INTEL_COMPILER)
#define	__IDSTRING(name,string)	__asm__(".ident\t\"" string "\"")
#else
# 594 "/usr/include/sys/cdefs.h" 3 4
/*
 * The following definition might not work well if used in header files,
 * but it should be better than nothing.  If you want a "do nothing"
 * version, then it should generate some harmless declaration, such as:
 *    #define	__IDSTRING(name,string)	struct __hack
 */
#define	__IDSTRING(name,string)	static const char name[] __unused = string
#endif
# 602 "/usr/include/sys/cdefs.h" 3 4

/*
 * Embed the rcs id of a source file in the resulting library.  Note that in
 * more recent ELF binutils, we use .ident allowing the ID to be stripped.
 * Usage:
 *	__FBSDID("$FreeBSD: head/sys/sys/cdefs.h 295041 2016-01-29 15:12:31Z br $");
 */
#ifndef	__FBSDID
#if !defined(lint) && !defined(STRIP_FBSDID)
#define	__FBSDID(s)	__IDSTRING(__CONCAT(__rcsid_,__LINE__),s)
#else
# 613 "/usr/include/sys/cdefs.h" 3 4
#define	__FBSDID(s)	struct __hack
#endif
# 615 "/usr/include/sys/cdefs.h" 3 4
#endif
# 616 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__RCSID
#ifndef	NO__RCSID
#define	__RCSID(s)	__IDSTRING(__CONCAT(__rcsid_,__LINE__),s)
#else
# 621 "/usr/include/sys/cdefs.h" 3 4
#define	__RCSID(s)	struct __hack
#endif
# 623 "/usr/include/sys/cdefs.h" 3 4
#endif
# 624 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__RCSID_SOURCE
#ifndef	NO__RCSID_SOURCE
#define	__RCSID_SOURCE(s)	__IDSTRING(__CONCAT(__rcsid_source_,__LINE__),s)
#else
# 629 "/usr/include/sys/cdefs.h" 3 4
#define	__RCSID_SOURCE(s)	struct __hack
#endif
# 631 "/usr/include/sys/cdefs.h" 3 4
#endif
# 632 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__SCCSID
#ifndef	NO__SCCSID
#define	__SCCSID(s)	__IDSTRING(__CONCAT(__sccsid_,__LINE__),s)
#else
# 637 "/usr/include/sys/cdefs.h" 3 4
#define	__SCCSID(s)	struct __hack
#endif
# 639 "/usr/include/sys/cdefs.h" 3 4
#endif
# 640 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__COPYRIGHT
#ifndef	NO__COPYRIGHT
#define	__COPYRIGHT(s)	__IDSTRING(__CONCAT(__copyright_,__LINE__),s)
#else
# 645 "/usr/include/sys/cdefs.h" 3 4
#define	__COPYRIGHT(s)	struct __hack
#endif
# 647 "/usr/include/sys/cdefs.h" 3 4
#endif
# 648 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__DECONST
#define	__DECONST(type, var)	((type)(__uintptr_t)(const void *)(var))
#endif
# 652 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__DEVOLATILE
#define	__DEVOLATILE(type, var)	((type)(__uintptr_t)(volatile void *)(var))
#endif
# 656 "/usr/include/sys/cdefs.h" 3 4

#ifndef	__DEQUALIFY
#define	__DEQUALIFY(type, var)	((type)(__uintptr_t)(const volatile void *)(var))
#endif
# 660 "/usr/include/sys/cdefs.h" 3 4

/*-
 * The following definitions are an extension of the behavior originally
 * implemented in <sys/_posix.h>, but with a different level of granularity.
 * POSIX.1 requires that the macros we test be defined before any standard
 * header file is included.
 *
 * Here's a quick run-down of the versions:
 *  defined(_POSIX_SOURCE)		1003.1-1988
 *  _POSIX_C_SOURCE == 1		1003.1-1990
 *  _POSIX_C_SOURCE == 2		1003.2-1992 C Language Binding Option
 *  _POSIX_C_SOURCE == 199309		1003.1b-1993
 *  _POSIX_C_SOURCE == 199506		1003.1c-1995, 1003.1i-1995,
 *					and the omnibus ISO/IEC 9945-1: 1996
 *  _POSIX_C_SOURCE == 200112		1003.1-2001
 *  _POSIX_C_SOURCE == 200809		1003.1-2008
 *
 * In addition, the X/Open Portability Guide, which is now the Single UNIX
 * Specification, defines a feature-test macro which indicates the version of
 * that specification, and which subsumes _POSIX_C_SOURCE.
 *
 * Our macros begin with two underscores to avoid namespace screwage.
 */

/* Deal with IEEE Std. 1003.1-1990, in which _POSIX_C_SOURCE == 1. */
#if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE == 1
#undef _POSIX_C_SOURCE		/* Probably illegal, but beyond caring now. */
#define	_POSIX_C_SOURCE		199009
#endif
# 689 "/usr/include/sys/cdefs.h" 3 4

/* Deal with IEEE Std. 1003.2-1992, in which _POSIX_C_SOURCE == 2. */
#if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE == 2
#undef _POSIX_C_SOURCE
#define	_POSIX_C_SOURCE		199209
#endif
# 695 "/usr/include/sys/cdefs.h" 3 4

/* Deal with various X/Open Portability Guides and Single UNIX Spec. */
#ifdef _XOPEN_SOURCE
#if _XOPEN_SOURCE - 0 >= 700
#define	__XSI_VISIBLE		700
#undef _POSIX_C_SOURCE
#define	_POSIX_C_SOURCE		200809
#elif _XOPEN_SOURCE - 0 >= 600
# 703 "/usr/include/sys/cdefs.h" 3 4
#define	__XSI_VISIBLE		600
#undef _POSIX_C_SOURCE
#define	_POSIX_C_SOURCE		200112
#elif _XOPEN_SOURCE - 0 >= 500
# 707 "/usr/include/sys/cdefs.h" 3 4
#define	__XSI_VISIBLE		500
#undef _POSIX_C_SOURCE
#define	_POSIX_C_SOURCE		199506
#endif
# 711 "/usr/include/sys/cdefs.h" 3 4
#endif
# 712 "/usr/include/sys/cdefs.h" 3 4

/*
 * Deal with all versions of POSIX.  The ordering relative to the tests above is
 * important.
 */
#if defined(_POSIX_SOURCE) && !defined(_POSIX_C_SOURCE)
#define	_POSIX_C_SOURCE		198808
#endif
# 720 "/usr/include/sys/cdefs.h" 3 4
#ifdef _POSIX_C_SOURCE
#if _POSIX_C_SOURCE >= 200809
#define	__POSIX_VISIBLE		200809
#define	__ISO_C_VISIBLE		1999
#elif _POSIX_C_SOURCE >= 200112
# 725 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		200112
#define	__ISO_C_VISIBLE		1999
#elif _POSIX_C_SOURCE >= 199506
# 728 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		199506
#define	__ISO_C_VISIBLE		1990
#elif _POSIX_C_SOURCE >= 199309
# 731 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		199309
#define	__ISO_C_VISIBLE		1990
#elif _POSIX_C_SOURCE >= 199209
# 734 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		199209
#define	__ISO_C_VISIBLE		1990
#elif _POSIX_C_SOURCE >= 199009
# 737 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		199009
#define	__ISO_C_VISIBLE		1990
#else
# 740 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		198808
#define	__ISO_C_VISIBLE		0
#endif /* _POSIX_C_SOURCE */
# 743 "/usr/include/sys/cdefs.h" 3 4
#else
# 744 "/usr/include/sys/cdefs.h" 3 4
/*-
 * Deal with _ANSI_SOURCE:
 * If it is defined, and no other compilation environment is explicitly
 * requested, then define our internal feature-test macros to zero.  This
 * makes no difference to the preprocessor (undefined symbols in preprocessing
 * expressions are defined to have value zero), but makes it more convenient for
 * a test program to print out the values.
 *
 * If a program mistakenly defines _ANSI_SOURCE and some other macro such as
 * _POSIX_C_SOURCE, we will assume that it wants the broader compilation
 * environment (and in fact we will never get here).
 */
#if defined(_ANSI_SOURCE)	/* Hide almost everything. */
#define	__POSIX_VISIBLE		0
#define	__XSI_VISIBLE		0
#define	__BSD_VISIBLE		0
#define	__ISO_C_VISIBLE		1990
#elif defined(_C99_SOURCE)	/* Localism to specify strict C99 env. */
# 762 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		0
#define	__XSI_VISIBLE		0
#define	__BSD_VISIBLE		0
#define	__ISO_C_VISIBLE		1999
#elif defined(_C11_SOURCE)	/* Localism to specify strict C11 env. */
# 767 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		0
#define	__XSI_VISIBLE		0
#define	__BSD_VISIBLE		0
#define	__ISO_C_VISIBLE		2011
#else				/* Default environment: show everything. */
# 772 "/usr/include/sys/cdefs.h" 3 4
#define	__POSIX_VISIBLE		200809
#define	__XSI_VISIBLE		700
#define	__BSD_VISIBLE		1
#define	__ISO_C_VISIBLE		2011
#endif
# 777 "/usr/include/sys/cdefs.h" 3 4
#endif
# 778 "/usr/include/sys/cdefs.h" 3 4

#if defined(__mips) || defined(__powerpc64__) || defined(__riscv__)
#define	__NO_TLS 1
#endif
# 782 "/usr/include/sys/cdefs.h" 3 4

/*
 * Type Safety Checking
 *
 * Clang provides additional attributes to enable checking type safety
 * properties that cannot be enforced by the C type system. 
 */

#if __has_attribute(__argument_with_type_tag__) && \
    __has_attribute(__type_tag_for_datatype__) && !defined(lint)
#define	__arg_type_tag(arg_kind, arg_idx, type_tag_idx) \
	    __attribute__((__argument_with_type_tag__(arg_kind, arg_idx, type_tag_idx)))
#define	__datatype_type_tag(kind, type) \
	    __attribute__((__type_tag_for_datatype__(kind, type)))
#else
# 797 "/usr/include/sys/cdefs.h" 3 4
#define	__arg_type_tag(arg_kind, arg_idx, type_tag_idx)
#define	__datatype_type_tag(kind, type)
#endif
# 800 "/usr/include/sys/cdefs.h" 3 4

/*
 * Lock annotations.
 *
 * Clang provides support for doing basic thread-safety tests at
 * compile-time, by marking which locks will/should be held when
 * entering/leaving a functions.
 *
 * Furthermore, it is also possible to annotate variables and structure
 * members to enforce that they are only accessed when certain locks are
 * held.
 */

#if __has_extension(c_thread_safety_attributes)
#define	__lock_annotate(x)	__attribute__((x))
#else
# 816 "/usr/include/sys/cdefs.h" 3 4
#define	__lock_annotate(x)
#endif
# 818 "/usr/include/sys/cdefs.h" 3 4

/* Structure implements a lock. */
#define	__lockable		__lock_annotate(lockable)

/* Function acquires an exclusive or shared lock. */
#define	__locks_exclusive(...) \
	__lock_annotate(exclusive_lock_function(__VA_ARGS__))
#define	__locks_shared(...) \
	__lock_annotate(shared_lock_function(__VA_ARGS__))

/* Function attempts to acquire an exclusive or shared lock. */
#define	__trylocks_exclusive(...) \
	__lock_annotate(exclusive_trylock_function(__VA_ARGS__))
#define	__trylocks_shared(...) \
	__lock_annotate(shared_trylock_function(__VA_ARGS__))

/* Function releases a lock. */
#define	__unlocks(...)		__lock_annotate(unlock_function(__VA_ARGS__))

/* Function asserts that an exclusive or shared lock is held. */
#define	__asserts_exclusive(...) \
	__lock_annotate(assert_exclusive_lock(__VA_ARGS__))
#define	__asserts_shared(...) \
	__lock_annotate(assert_shared_lock(__VA_ARGS__))

/* Function requires that an exclusive or shared lock is or is not held. */
#define	__requires_exclusive(...) \
	__lock_annotate(exclusive_locks_required(__VA_ARGS__))
#define	__requires_shared(...) \
	__lock_annotate(shared_locks_required(__VA_ARGS__))
#define	__requires_unlocked(...) \
	__lock_annotate(locks_excluded(__VA_ARGS__))

/* Function should not be analyzed. */
#define	__no_lock_analysis	__lock_annotate(no_thread_safety_analysis)

/* Guard variables and structure members by lock. */
#define	__guarded_by(x)		__lock_annotate(guarded_by(x))
#define	__pt_guarded_by(x)	__lock_annotate(pt_guarded_by(x))

#endif /* !_SYS_CDEFS_H_ */
# 859 "/usr/include/sys/cdefs.h" 3 4
# 38 "/usr/include/stddef.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/stddef.h" 3 4
# 1 "/usr/include/sys/_null.h" 1 3 4
/*-
 * Copyright (c) 2003 Marcel Moolenaar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/_null.h 228918 2011-12-27 21:36:31Z theraven $
 */

#ifndef NULL

#if !defined(__cplusplus)
#define	NULL	((void *)0)
#else
# 34 "/usr/include/sys/_null.h" 3 4
#if __cplusplus >= 201103L
#define	NULL	nullptr
#elif defined(__GNUG__) && defined(__GNUC__) && __GNUC__ >= 4
# 37 "/usr/include/sys/_null.h" 3 4
#define	NULL	__null
#else
# 39 "/usr/include/sys/_null.h" 3 4
#if defined(__LP64__)
#define	NULL	(0L)
#else
# 42 "/usr/include/sys/_null.h" 3 4
#define	NULL	0
#endif	/* __LP64__ */
# 44 "/usr/include/sys/_null.h" 3 4
#endif	/* __GNUG__ */
# 45 "/usr/include/sys/_null.h" 3 4
#endif	/* !__cplusplus */
# 46 "/usr/include/sys/_null.h" 3 4

#endif
# 48 "/usr/include/sys/_null.h" 3 4
# 39 "/usr/include/stddef.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 39 "/usr/include/stddef.h" 3 4
# 1 "/usr/include/sys/_types.h" 1 3 4
/*-
 * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/_types.h 297000 2016-03-18 01:28:41Z jhibbits $
 */

#ifndef _SYS__TYPES_H_
#define _SYS__TYPES_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 32 "/usr/include/sys/_types.h" 3 4
# 33 "/usr/include/sys/_types.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/_types.h>
#endif /* expanded by -frewrite-includes */
# 33 "/usr/include/sys/_types.h" 3 4
# 1 "/usr/include/machine/_types.h" 1 3 4
/*-
 * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org>
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	From: @(#)ansi.h	8.2 (Berkeley) 1/4/94
 *	From: @(#)types.h	8.3 (Berkeley) 1/5/94
 * $FreeBSD: head/sys/arm/include/_types.h 283812 2015-05-31 10:51:06Z andrew $
 */

#ifndef _MACHINE__TYPES_H_
#define	_MACHINE__TYPES_H_

#ifndef _SYS_CDEFS_H_
#error this file needs sys/cdefs.h as a prerequisite
#endif
# 45 "/usr/include/machine/_types.h" 3 4

/*
 * Basic types upon which most other types are built.
 */
typedef	signed char		__int8_t;
typedef	unsigned char		__uint8_t;
typedef	short			__int16_t;
typedef	unsigned short		__uint16_t;
typedef	int			__int32_t;
typedef	unsigned int		__uint32_t;
#ifndef lint
__extension__
#endif
# 58 "/usr/include/machine/_types.h" 3 4
/* LONGLONG */
typedef	long long		__int64_t;
#ifndef lint
__extension__
#endif
# 63 "/usr/include/machine/_types.h" 3 4
/* LONGLONG */
typedef	unsigned long long	__uint64_t;

/*
 * Standard type definitions.
 */
typedef	__uint32_t	__clock_t;		/* clock()... */
typedef	__int32_t	__critical_t;
typedef	double		__double_t;
typedef	float		__float_t;
typedef	__int32_t	__intfptr_t;
typedef	__int64_t	__intmax_t;
typedef	__int32_t	__intptr_t;
typedef	__int32_t	__int_fast8_t;
typedef	__int32_t	__int_fast16_t;
typedef	__int32_t	__int_fast32_t;
typedef	__int64_t	__int_fast64_t;
typedef	__int8_t	__int_least8_t;
typedef	__int16_t	__int_least16_t;
typedef	__int32_t	__int_least32_t;
typedef	__int64_t	__int_least64_t;
typedef	__int32_t	__ptrdiff_t;		/* ptr1 - ptr2 */
typedef	__int32_t	__register_t;
typedef	__int32_t	__segsz_t;		/* segment size (in pages) */
typedef	__uint32_t	__size_t;		/* sizeof() */
typedef	__int32_t	__ssize_t;		/* byte count or error */
typedef	__int64_t	__time_t;		/* time()... */
typedef	__uint32_t	__uintfptr_t;
typedef	__uint64_t	__uintmax_t;
typedef	__uint32_t	__uintptr_t;
typedef	__uint32_t	__uint_fast8_t;
typedef	__uint32_t	__uint_fast16_t;
typedef	__uint32_t	__uint_fast32_t;
typedef	__uint64_t	__uint_fast64_t;
typedef	__uint8_t	__uint_least8_t;
typedef	__uint16_t	__uint_least16_t;
typedef	__uint32_t	__uint_least32_t;
typedef	__uint64_t	__uint_least64_t;
typedef	__uint32_t	__u_register_t;
typedef	__uint32_t	__vm_offset_t;
typedef	__int64_t	__vm_ooffset_t;
typedef	__uint32_t	__vm_paddr_t;
typedef	__uint64_t	__vm_pindex_t;
typedef	__uint32_t	__vm_size_t;

typedef	unsigned int	___wchar_t;
#define	__WCHAR_MIN	0		/* min value for a wchar_t */
#define	__WCHAR_MAX	__UINT_MAX	/* max value for a wchar_t */

/*
 * Unusual type definitions.
 */
#ifdef __GNUCLIKE_BUILTIN_VARARGS
typedef __builtin_va_list	__va_list;	/* internally known to gcc */
#else
# 118 "/usr/include/machine/_types.h" 3 4
typedef	char *			__va_list;
#endif /* __GNUCLIKE_BUILTIN_VARARGS */
# 120 "/usr/include/machine/_types.h" 3 4
#if defined(__GNUCLIKE_BUILTIN_VAALIST) && !defined(__GNUC_VA_LIST) \
    && !defined(__NO_GNUC_VA_LIST)
#define __GNUC_VA_LIST
typedef __va_list		__gnuc_va_list;	/* compatibility w/GNU headers*/
#endif
# 125 "/usr/include/machine/_types.h" 3 4

#endif /* !_MACHINE__TYPES_H_ */
# 127 "/usr/include/machine/_types.h" 3 4
# 34 "/usr/include/sys/_types.h" 2 3 4

/*
 * Standard type definitions.
 */
typedef	__int32_t	__blksize_t;	/* file block size */
typedef	__int64_t	__blkcnt_t;	/* file block count */
typedef	__int32_t	__clockid_t;	/* clock_gettime()... */
typedef	__uint32_t	__fflags_t;	/* file flags */
typedef	__uint64_t	__fsblkcnt_t;
typedef	__uint64_t	__fsfilcnt_t;
typedef	__uint32_t	__gid_t;
typedef	__int64_t	__id_t;		/* can hold a gid_t, pid_t, or uid_t */
typedef	__uint32_t	__ino_t;	/* inode number */
typedef	long		__key_t;	/* IPC key (for Sys V IPC) */
typedef	__int32_t	__lwpid_t;	/* Thread ID (a.k.a. LWP) */
typedef	__uint16_t	__mode_t;	/* permissions */
typedef	int		__accmode_t;	/* access permissions */
typedef	int		__nl_item;
typedef	__uint16_t	__nlink_t;	/* link count */
typedef	__int64_t	__off_t;	/* file offset */
typedef	__int32_t	__pid_t;	/* process [group] */
typedef	__int64_t	__rlim_t;	/* resource limit - intentionally */
					/* signed, because of legacy code */
					/* that uses -1 for RLIM_INFINITY */
typedef	__uint8_t	__sa_family_t;
typedef	__uint32_t	__socklen_t;
typedef	long		__suseconds_t;	/* microseconds (signed) */
typedef	struct __timer	*__timer_t;	/* timer_gettime()... */
typedef	struct __mq	*__mqd_t;	/* mq_open()... */
typedef	__uint32_t	__uid_t;
typedef	unsigned int	__useconds_t;	/* microseconds (unsigned) */
typedef	int		__cpuwhich_t;	/* which parameter for cpuset. */
typedef	int		__cpulevel_t;	/* level parameter for cpuset. */
typedef int		__cpusetid_t;	/* cpuset identifier. */

/*
 * Unusual type definitions.
 */
/*
 * rune_t is declared to be an ``int'' instead of the more natural
 * ``unsigned long'' or ``long''.  Two things are happening here.  It is not
 * unsigned so that EOF (-1) can be naturally assigned to it and used.  Also,
 * it looks like 10646 will be a 31 bit standard.  This means that if your
 * ints cannot hold 32 bits, you will be in trouble.  The reason an int was
 * chosen over a long is that the is*() and to*() routines take ints (says
 * ANSI C), but they use __ct_rune_t instead of int.
 *
 * NOTE: rune_t is not covered by ANSI nor other standards, and should not
 * be instantiated outside of lib/libc/locale.  Use wchar_t.  wint_t and
 * rune_t must be the same type.  Also, wint_t should be able to hold all
 * members of the largest character set plus one extra value (WEOF), and
 * must be at least 16 bits.
 */
typedef	int		__ct_rune_t;	/* arg type for ctype funcs */
typedef	__ct_rune_t	__rune_t;	/* rune_t (see above) */
typedef	__ct_rune_t	__wint_t;	/* wint_t (see above) */

/* Clang already provides these types as built-ins, but only in C++ mode. */
#if !defined(__clang__) || !defined(__cplusplus)
typedef	__uint_least16_t __char16_t;
typedef	__uint_least32_t __char32_t;
#endif
# 96 "/usr/include/sys/_types.h" 3 4
/* In C++11, char16_t and char32_t are built-in types. */
#if defined(__cplusplus) && __cplusplus >= 201103L
#define	_CHAR16_T_DECLARED
#define	_CHAR32_T_DECLARED
#endif
# 101 "/usr/include/sys/_types.h" 3 4

typedef	__uint32_t	__dev_t;	/* device number */

typedef	__uint32_t	__fixpt_t;	/* fixed point number */

/*
 * mbstate_t is an opaque object to keep conversion state during multibyte
 * stream conversions.
 */
typedef union {
	char		__mbstate8[128];
	__int64_t	_mbstateL;	/* for alignment */
} __mbstate_t;

typedef __uintmax_t     __rman_res_t;

#endif /* !_SYS__TYPES_H_ */
# 118 "/usr/include/sys/_types.h" 3 4
# 40 "/usr/include/stddef.h" 2 3 4

#ifndef _PTRDIFF_T_DECLARED
typedef	__ptrdiff_t	ptrdiff_t;
#define	_PTRDIFF_T_DECLARED
#endif
# 45 "/usr/include/stddef.h" 3 4

#if __BSD_VISIBLE
#ifndef _RUNE_T_DECLARED
typedef	__rune_t	rune_t;
#define	_RUNE_T_DECLARED
#endif
# 51 "/usr/include/stddef.h" 3 4
#endif
# 52 "/usr/include/stddef.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 57 "/usr/include/stddef.h" 3 4

#ifndef	__cplusplus
#ifndef _WCHAR_T_DECLARED
typedef	___wchar_t	wchar_t;
#define	_WCHAR_T_DECLARED
#endif
# 63 "/usr/include/stddef.h" 3 4
#endif
# 64 "/usr/include/stddef.h" 3 4

#define	offsetof(type, member)	__offsetof(type, member)

#endif /* _STDDEF_H_ */
# 68 "/usr/include/stddef.h" 3 4
# 26 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 26 "beam/beam_emu.c"
# 1 "beam/sys.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __SYS_H__
#define __SYS_H__

#ifdef ERTS_INLINE
#  ifndef ERTS_CAN_INLINE
#    define ERTS_CAN_INLINE 1
#  endif
# 28 "beam/sys.h"
#else
# 29 "beam/sys.h"
#  if defined(__GNUC__)
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline__
#  elif defined(__WIN32__)
# 33 "beam/sys.h"
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline
#  else
# 36 "beam/sys.h"
#    define ERTS_CAN_INLINE 0
#    define ERTS_INLINE
#  endif
# 39 "beam/sys.h"
#endif
# 40 "beam/sys.h"

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#  undef ERTS_CAN_INLINE
#  define ERTS_CAN_INLINE 0
#  undef ERTS_INLINE
#  define ERTS_INLINE
#endif
# 47 "beam/sys.h"

#if ERTS_CAN_INLINE
#define ERTS_GLB_INLINE static ERTS_INLINE
#else
# 51 "beam/sys.h"
#define ERTS_GLB_INLINE
#endif
# 53 "beam/sys.h"

#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
#else
# 57 "beam/sys.h"
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
# 59 "beam/sys.h"

#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
#  define NO_FPE_SIGNALS
#endif
# 63 "beam/sys.h"

#ifdef DISABLE_CHILD_WAITER_THREAD
#undef ENABLE_CHILD_WAITER_THREAD
#endif
# 67 "beam/sys.h"

#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
#undef ENABLE_CHILD_WAITER_THREAD
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
# 72 "beam/sys.h"

#define ERTS_I64_LITERAL(X) X##LL

#if defined (__WIN32__)
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_sys.h"
#endif /* expanded by -frewrite-includes */
# 76 "beam/sys.h"
# 77 "beam/sys.h"
#elif defined (__OSE__)
# 78 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_ose_sys.h"
#endif /* expanded by -frewrite-includes */
# 78 "beam/sys.h"
# 79 "beam/sys.h"
#else 
# 80 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_unix_sys.h"
#endif /* expanded by -frewrite-includes */
# 80 "beam/sys.h"
# 1 "sys/unix/erl_unix_sys.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 1997-2011. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 *
 * This file handles differences between different Unix systems.
 * This should be the only place with conditional compilation
 * depending on the type of OS.
 */

#ifndef _ERL_UNIX_SYS_H
#define _ERL_UNIX_SYS_H

#if 0 /* expanded by -frewrite-includes */
#include <stdio.h>
#endif /* expanded by -frewrite-includes */
# 28 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/stdio.h" 1 3 4
/*-
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * Chris Torek.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)stdio.h	8.5 (Berkeley) 4/29/95
 * $FreeBSD: head/include/stdio.h 290110 2015-10-28 14:40:02Z ache $
 */

#ifndef	_STDIO_H_
#define	_STDIO_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 39 "/usr/include/stdio.h" 3 4
# 40 "/usr/include/stdio.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 40 "/usr/include/stdio.h" 3 4
# 41 "/usr/include/stdio.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/stdio.h" 3 4
# 42 "/usr/include/stdio.h" 3 4

typedef	__off_t		fpos_t;

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 49 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE >= 200809
#ifndef _OFF_T_DECLARED
#define	_OFF_T_DECLARED
typedef	__off_t		off_t;
#endif
# 55 "/usr/include/stdio.h" 3 4
#ifndef _SSIZE_T_DECLARED
#define	_SSIZE_T_DECLARED
typedef	__ssize_t	ssize_t;
#endif
# 59 "/usr/include/stdio.h" 3 4
#endif
# 60 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#ifndef _VA_LIST_DECLARED
typedef	__va_list	va_list;
#define	_VA_LIST_DECLARED
#endif
# 66 "/usr/include/stdio.h" 3 4
#endif
# 67 "/usr/include/stdio.h" 3 4

#define	_FSTDIO			/* Define for new stdio with functions. */

/*
 * NB: to fit things in six character monocase externals, the stdio
 * code uses the prefix `__s' for stdio objects, typically followed
 * by a three-character attempt at a mnemonic.
 */

/* stdio buffers */
struct __sbuf {
	unsigned char *_base;
	int	_size;
};

/*
 * stdio state variables.
 *
 * The following always hold:
 *
 *	if (_flags&(__SLBF|__SWR)) == (__SLBF|__SWR),
 *		_lbfsize is -_bf._size, else _lbfsize is 0
 *	if _flags&__SRD, _w is 0
 *	if _flags&__SWR, _r is 0
 *
 * This ensures that the getc and putc macros (or inline functions) never
 * try to write or read from a file that is in `read' or `write' mode.
 * (Moreover, they can, and do, automatically switch from read mode to
 * write mode, and back, on "r+" and "w+" files.)
 *
 * _lbfsize is used only to make the inline line-buffered output stream
 * code as compact as possible.
 *
 * _ub, _up, and _ur are used when ungetc() pushes back more characters
 * than fit in the current _bf, or when ungetc() pushes back a character
 * that does not match the previous one in _bf.  When this happens,
 * _ub._base becomes non-nil (i.e., a stream has ungetc() data iff
 * _ub._base!=NULL) and _up and _ur save the current values of _p and _r.
 *
 * Certain members of __sFILE are accessed directly via macros or
 * inline functions.  To preserve ABI compat, these members must not
 * be disturbed.  These members are marked below with (*).
 */
struct __sFILE {
	unsigned char *_p;	/* (*) current position in (some) buffer */
	int	_r;		/* (*) read space left for getc() */
	int	_w;		/* (*) write space left for putc() */
	short	_flags;		/* (*) flags, below; this FILE is free if 0 */
	short	_file;		/* (*) fileno, if Unix descriptor, else -1 */
	struct	__sbuf _bf;	/* (*) the buffer (at least 1 byte, if !NULL) */
	int	_lbfsize;	/* (*) 0 or -_bf._size, for inline putc */

	/* operations */
	void	*_cookie;	/* (*) cookie passed to io functions */
	int	(*_close)(void *);
	int	(*_read)(void *, char *, int);
	fpos_t	(*_seek)(void *, fpos_t, int);
	int	(*_write)(void *, const char *, int);

	/* separate buffer for long sequences of ungetc() */
	struct	__sbuf _ub;	/* ungetc buffer */
	unsigned char	*_up;	/* saved _p when _p is doing ungetc data */
	int	_ur;		/* saved _r when _r is counting ungetc data */

	/* tricks to meet minimum requirements even when malloc() fails */
	unsigned char _ubuf[3];	/* guarantee an ungetc() buffer */
	unsigned char _nbuf[1];	/* guarantee a getc() buffer */

	/* separate buffer for fgetln() when line crosses buffer boundary */
	struct	__sbuf _lb;	/* buffer for fgetln() */

	/* Unix stdio files get aligned to block boundaries on fseek() */
	int	_blksize;	/* stat.st_blksize (may be != _bf._size) */
	fpos_t	_offset;	/* current lseek offset */

	struct pthread_mutex *_fl_mutex;	/* used for MT-safety */
	struct pthread *_fl_owner;	/* current owner */
	int	_fl_count;	/* recursive lock count */
	int	_orientation;	/* orientation for fwide() */
	__mbstate_t _mbstate;	/* multibyte conversion state */
	int	_flags2;	/* additional flags */
};
#ifndef _STDFILE_DECLARED
#define _STDFILE_DECLARED
typedef struct __sFILE FILE;
#endif
# 153 "/usr/include/stdio.h" 3 4
#ifndef _STDSTREAM_DECLARED
__BEGIN_DECLS
extern FILE *__stdinp;
extern FILE *__stdoutp;
extern FILE *__stderrp;
__END_DECLS
#define	_STDSTREAM_DECLARED
#endif
# 161 "/usr/include/stdio.h" 3 4

#define	__SLBF	0x0001		/* line buffered */
#define	__SNBF	0x0002		/* unbuffered */
#define	__SRD	0x0004		/* OK to read */
#define	__SWR	0x0008		/* OK to write */
	/* RD and WR are never simultaneously asserted */
#define	__SRW	0x0010		/* open for reading & writing */
#define	__SEOF	0x0020		/* found EOF */
#define	__SERR	0x0040		/* found error */
#define	__SMBF	0x0080		/* _bf._base is from malloc */
#define	__SAPP	0x0100		/* fdopen()ed in append mode */
#define	__SSTR	0x0200		/* this is an sprintf/snprintf string */
#define	__SOPT	0x0400		/* do fseek() optimization */
#define	__SNPT	0x0800		/* do not do fseek() optimization */
#define	__SOFF	0x1000		/* set iff _offset is in fact correct */
#define	__SMOD	0x2000		/* true => fgetln modified _p text */
#define	__SALC	0x4000		/* allocate string space dynamically */
#define	__SIGN	0x8000		/* ignore this file in _fwalk */

#define	__S2OAP	0x0001		/* O_APPEND mode is set */

/*
 * The following three definitions are for ANSI C, which took them
 * from System V, which brilliantly took internal interface macros and
 * made them official arguments to setvbuf(), without renaming them.
 * Hence, these ugly _IOxxx names are *supposed* to appear in user code.
 *
 * Although numbered as their counterparts above, the implementation
 * does not rely on this.
 */
#define	_IOFBF	0		/* setvbuf should set fully buffered */
#define	_IOLBF	1		/* setvbuf should set line buffered */
#define	_IONBF	2		/* setvbuf should set unbuffered */

#define	BUFSIZ	1024		/* size of buffer used by setbuf */
#define	EOF	(-1)

/*
 * FOPEN_MAX is a minimum maximum, and is the number of streams that
 * stdio can provide without attempting to allocate further resources
 * (which could fail).  Do not use this for anything.
 */
				/* must be == _POSIX_STREAM_MAX <limits.h> */
#ifndef FOPEN_MAX
#define	FOPEN_MAX	20	/* must be <= OPEN_MAX <sys/syslimits.h> */
#endif
# 207 "/usr/include/stdio.h" 3 4
#define	FILENAME_MAX	1024	/* must be <= PATH_MAX <sys/syslimits.h> */

/* System V/ANSI C; this is the wrong way to do this, do *not* use these. */
#if __XSI_VISIBLE
#define	P_tmpdir	"/tmp/"
#endif
# 213 "/usr/include/stdio.h" 3 4
#define	L_tmpnam	1024	/* XXX must be == PATH_MAX */
#define	TMP_MAX		308915776

#ifndef SEEK_SET
#define	SEEK_SET	0	/* set file offset to offset */
#endif
# 219 "/usr/include/stdio.h" 3 4
#ifndef SEEK_CUR
#define	SEEK_CUR	1	/* set file offset to current plus offset */
#endif
# 222 "/usr/include/stdio.h" 3 4
#ifndef SEEK_END
#define	SEEK_END	2	/* set file offset to EOF plus offset */
#endif
# 225 "/usr/include/stdio.h" 3 4

#define	stdin	__stdinp
#define	stdout	__stdoutp
#define	stderr	__stderrp

__BEGIN_DECLS
#ifdef _XLOCALE_H_
#if 0 /* expanded by -frewrite-includes */
#include <xlocale/_stdio.h>
#endif /* expanded by -frewrite-includes */
# 232 "/usr/include/stdio.h" 3 4
# 233 "/usr/include/stdio.h" 3 4
#endif
# 234 "/usr/include/stdio.h" 3 4
/*
 * Functions defined in ANSI C standard.
 */
void	 clearerr(FILE *);
int	 fclose(FILE *);
int	 feof(FILE *);
int	 ferror(FILE *);
int	 fflush(FILE *);
int	 fgetc(FILE *);
int	 fgetpos(FILE * __restrict, fpos_t * __restrict);
char	*fgets(char * __restrict, int, FILE * __restrict);
FILE	*fopen(const char * __restrict, const char * __restrict);
int	 fprintf(FILE * __restrict, const char * __restrict, ...);
int	 fputc(int, FILE *);
int	 fputs(const char * __restrict, FILE * __restrict);
size_t	 fread(void * __restrict, size_t, size_t, FILE * __restrict);
FILE	*freopen(const char * __restrict, const char * __restrict, FILE * __restrict);
int	 fscanf(FILE * __restrict, const char * __restrict, ...);
int	 fseek(FILE *, long, int);
int	 fsetpos(FILE *, const fpos_t *);
long	 ftell(FILE *);
size_t	 fwrite(const void * __restrict, size_t, size_t, FILE * __restrict);
int	 getc(FILE *);
int	 getchar(void);
char	*gets(char *);
void	 perror(const char *);
int	 printf(const char * __restrict, ...);
int	 putc(int, FILE *);
int	 putchar(int);
int	 puts(const char *);
int	 remove(const char *);
int	 rename(const char *, const char *);
void	 rewind(FILE *);
int	 scanf(const char * __restrict, ...);
void	 setbuf(FILE * __restrict, char * __restrict);
int	 setvbuf(FILE * __restrict, char * __restrict, int, size_t);
int	 sprintf(char * __restrict, const char * __restrict, ...);
int	 sscanf(const char * __restrict, const char * __restrict, ...);
FILE	*tmpfile(void);
char	*tmpnam(char *);
int	 ungetc(int, FILE *);
int	 vfprintf(FILE * __restrict, const char * __restrict,
	    __va_list);
int	 vprintf(const char * __restrict, __va_list);
int	 vsprintf(char * __restrict, const char * __restrict,
	    __va_list);

#if __ISO_C_VISIBLE >= 1999
int	 snprintf(char * __restrict, size_t, const char * __restrict,
	    ...) __printflike(3, 4);
int	 vfscanf(FILE * __restrict, const char * __restrict, __va_list)
	    __scanflike(2, 0);
int	 vscanf(const char * __restrict, __va_list) __scanflike(1, 0);
int	 vsnprintf(char * __restrict, size_t, const char * __restrict,
	    __va_list) __printflike(3, 0);
int	 vsscanf(const char * __restrict, const char * __restrict, __va_list)
	    __scanflike(2, 0);
#endif
# 292 "/usr/include/stdio.h" 3 4

/*
 * Functions defined in all versions of POSIX 1003.1.
 */
#if __BSD_VISIBLE || (__POSIX_VISIBLE && __POSIX_VISIBLE <= 199506)
#define	L_cuserid	17	/* size for cuserid(3); MAXLOGNAME, legacy */
#endif
# 299 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE
#define	L_ctermid	1024	/* size for ctermid(3); PATH_MAX */

char	*ctermid(char *);
FILE	*fdopen(int, const char *);
int	 fileno(FILE *);
#endif /* __POSIX_VISIBLE */
# 307 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE >= 199209
int	 pclose(FILE *);
FILE	*popen(const char *, const char *);
#endif
# 312 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE >= 199506
int	 ftrylockfile(FILE *);
void	 flockfile(FILE *);
void	 funlockfile(FILE *);

/*
 * These are normally used through macros as defined below, but POSIX
 * requires functions as well.
 */
int	 getc_unlocked(FILE *);
int	 getchar_unlocked(void);
int	 putc_unlocked(int, FILE *);
int	 putchar_unlocked(int);
#endif
# 327 "/usr/include/stdio.h" 3 4
#if __BSD_VISIBLE
void	 clearerr_unlocked(FILE *);
int	 feof_unlocked(FILE *);
int	 ferror_unlocked(FILE *);
int	 fileno_unlocked(FILE *);
#endif
# 333 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE >= 200112
int	 fseeko(FILE *, __off_t, int);
__off_t	 ftello(FILE *);
#endif
# 338 "/usr/include/stdio.h" 3 4

#if __BSD_VISIBLE || __XSI_VISIBLE > 0 && __XSI_VISIBLE < 600
int	 getw(FILE *);
int	 putw(int, FILE *);
#endif /* BSD or X/Open before issue 6 */
# 343 "/usr/include/stdio.h" 3 4

#if __XSI_VISIBLE
char	*tempnam(const char *, const char *);
#endif
# 347 "/usr/include/stdio.h" 3 4

#if __POSIX_VISIBLE >= 200809
FILE	*fmemopen(void * __restrict, size_t, const char * __restrict);
ssize_t	 getdelim(char ** __restrict, size_t * __restrict, int,
	    FILE * __restrict);
FILE	*open_memstream(char **, size_t *);
int	 renameat(int, const char *, int, const char *);
int	 vdprintf(int, const char * __restrict, __va_list);

/*
 * Every programmer and his dog wrote functions called getline() and dprintf()
 * before POSIX.1-2008 came along and decided to usurp the names, so we
 * don't prototype them by default unless one of the following is true:
 *   a) the app has requested them specifically by defining _WITH_GETLINE or
 *      _WITH_DPRINTF, respectively
 *   b) the app has requested a POSIX.1-2008 environment via _POSIX_C_SOURCE
 *   c) the app defines a GNUism such as _BSD_SOURCE or _GNU_SOURCE
 */
#ifndef _WITH_GETLINE
#if defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define	_WITH_GETLINE
#elif defined(_POSIX_C_SOURCE)
# 369 "/usr/include/stdio.h" 3 4
#if _POSIX_C_SOURCE >= 200809
#define	_WITH_GETLINE
#endif
# 372 "/usr/include/stdio.h" 3 4
#endif
# 373 "/usr/include/stdio.h" 3 4
#endif
# 374 "/usr/include/stdio.h" 3 4

#ifdef _WITH_GETLINE
ssize_t	 getline(char ** __restrict, size_t * __restrict, FILE * __restrict);
#endif
# 378 "/usr/include/stdio.h" 3 4

#ifndef _WITH_DPRINTF
#if defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define	_WITH_DPRINTF
#elif defined(_POSIX_C_SOURCE)
# 383 "/usr/include/stdio.h" 3 4
#if _POSIX_C_SOURCE >= 200809
#define	_WITH_DPRINTF
#endif
# 386 "/usr/include/stdio.h" 3 4
#endif
# 387 "/usr/include/stdio.h" 3 4
#endif
# 388 "/usr/include/stdio.h" 3 4

#ifdef _WITH_DPRINTF
int	 (dprintf)(int, const char * __restrict, ...);
#endif
# 392 "/usr/include/stdio.h" 3 4

#endif /* __POSIX_VISIBLE >= 200809 */
# 394 "/usr/include/stdio.h" 3 4

/*
 * Routines that are purely local.
 */
#if __BSD_VISIBLE
int	 asprintf(char **, const char *, ...) __printflike(2, 3);
char	*ctermid_r(char *);
void	 fcloseall(void);
int	 fdclose(FILE *, int *);
char	*fgetln(FILE *, size_t *);
const char *fmtcheck(const char *, const char *) __format_arg(2);
int	 fpurge(FILE *);
void	 setbuffer(FILE *, char *, int);
int	 setlinebuf(FILE *);
int	 vasprintf(char **, const char *, __va_list)
	    __printflike(2, 0);

/*
 * The system error table contains messages for the first sys_nerr
 * positive errno values.  Use strerror() or strerror_r() from <string.h>
 * instead.
 */
extern const int sys_nerr;
extern const char * const sys_errlist[];

/*
 * Stdio function-access interface.
 */
FILE	*funopen(const void *,
	    int (*)(void *, char *, int),
	    int (*)(void *, const char *, int),
	    fpos_t (*)(void *, fpos_t, int),
	    int (*)(void *));
#define	fropen(cookie, fn) funopen(cookie, fn, 0, 0, 0)
#define	fwopen(cookie, fn) funopen(cookie, 0, fn, 0, 0)

/*
 * Portability hacks.  See <sys/types.h>.
 */
#ifndef _FTRUNCATE_DECLARED
#define	_FTRUNCATE_DECLARED
int	 ftruncate(int, __off_t);
#endif
# 437 "/usr/include/stdio.h" 3 4
#ifndef _LSEEK_DECLARED
#define	_LSEEK_DECLARED
__off_t	 lseek(int, __off_t, int);
#endif
# 441 "/usr/include/stdio.h" 3 4
#ifndef _MMAP_DECLARED
#define	_MMAP_DECLARED
void	*mmap(void *, size_t, int, int, int, __off_t);
#endif
# 445 "/usr/include/stdio.h" 3 4
#ifndef _TRUNCATE_DECLARED
#define	_TRUNCATE_DECLARED
int	 truncate(const char *, __off_t);
#endif
# 449 "/usr/include/stdio.h" 3 4
#endif /* __BSD_VISIBLE */
# 450 "/usr/include/stdio.h" 3 4

/*
 * Functions internal to the implementation.
 */
int	__srget(FILE *);
int	__swbuf(int, FILE *);

/*
 * The __sfoo macros are here so that we can
 * define function versions in the C library.
 */
#define	__sgetc(p) (--(p)->_r < 0 ? __srget(p) : (int)(*(p)->_p++))
#if defined(__GNUC__) && defined(__STDC__)
static __inline int __sputc(int _c, FILE *_p) {
	if (--_p->_w >= 0 || (_p->_w >= _p->_lbfsize && (char)_c != '\n'))
		return (*_p->_p++ = _c);
	else
		return (__swbuf(_c, _p));
}
#else
# 470 "/usr/include/stdio.h" 3 4
/*
 * This has been tuned to generate reasonable code on the vax using pcc.
 */
#define	__sputc(c, p) \
	(--(p)->_w < 0 ? \
		(p)->_w >= (p)->_lbfsize ? \
			(*(p)->_p = (c)), *(p)->_p != '\n' ? \
				(int)*(p)->_p++ : \
				__swbuf('\n', p) : \
			__swbuf((int)(c), p) : \
		(*(p)->_p = (c), (int)*(p)->_p++))
#endif
# 482 "/usr/include/stdio.h" 3 4

extern int __isthreaded;

#ifndef __cplusplus

#define	__sfeof(p)	(((p)->_flags & __SEOF) != 0)
#define	__sferror(p)	(((p)->_flags & __SERR) != 0)
#define	__sclearerr(p)	((void)((p)->_flags &= ~(__SERR|__SEOF)))
#define	__sfileno(p)	((p)->_file)


#define	feof(p)		(!__isthreaded ? __sfeof(p) : (feof)(p))
#define	ferror(p)	(!__isthreaded ? __sferror(p) : (ferror)(p))
#define	clearerr(p)	(!__isthreaded ? __sclearerr(p) : (clearerr)(p))

#if __POSIX_VISIBLE
#define	fileno(p)	(!__isthreaded ? __sfileno(p) : (fileno)(p))
#endif
# 500 "/usr/include/stdio.h" 3 4

#define	getc(fp)	(!__isthreaded ? __sgetc(fp) : (getc)(fp))
#define	putc(x, fp)	(!__isthreaded ? __sputc(x, fp) : (putc)(x, fp))

#define	getchar()	getc(stdin)
#define	putchar(x)	putc(x, stdout)

#if __BSD_VISIBLE
/*
 * See ISO/IEC 9945-1 ANSI/IEEE Std 1003.1 Second Edition 1996-07-12
 * B.8.2.7 for the rationale behind the *_unlocked() macros.
 */
#define	feof_unlocked(p)	__sfeof(p)
#define	ferror_unlocked(p)	__sferror(p)
#define	clearerr_unlocked(p)	__sclearerr(p)
#define	fileno_unlocked(p)	__sfileno(p)
#endif
# 517 "/usr/include/stdio.h" 3 4
#if __POSIX_VISIBLE >= 199506
#define	getc_unlocked(fp)	__sgetc(fp)
#define	putc_unlocked(x, fp)	__sputc(x, fp)

#define	getchar_unlocked()	getc_unlocked(stdin)
#define	putchar_unlocked(x)	putc_unlocked(x, stdout)
#endif
# 524 "/usr/include/stdio.h" 3 4
#endif /* __cplusplus */
# 525 "/usr/include/stdio.h" 3 4

__END_DECLS
#endif /* !_STDIO_H_ */
# 528 "/usr/include/stdio.h" 3 4
# 29 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <math.h>
#endif /* expanded by -frewrite-includes */
# 29 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/math.h" 1 3 4
/*
 * ====================================================
 * Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
 *
 * Developed at SunPro, a Sun Microsystems, Inc. business.
 * Permission to use, copy, modify, and distribute this
 * software is freely granted, provided that this notice
 * is preserved.
 * ====================================================
 */

/*
 * from: @(#)fdlibm.h 5.1 93/09/24
 * $FreeBSD: head/lib/msun/src/math.h 271651 2014-09-15 23:21:57Z kargl $
 */

#ifndef _MATH_H_
#define	_MATH_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 20 "/usr/include/math.h" 3 4
# 21 "/usr/include/math.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 21 "/usr/include/math.h" 3 4
# 22 "/usr/include/math.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/_limits.h>
#endif /* expanded by -frewrite-includes */
# 22 "/usr/include/math.h" 3 4
# 1 "/usr/include/machine/_limits.h" 1 3 4
/*-
 * Copyright (c) 1988, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)limits.h	8.3 (Berkeley) 1/4/94
 * $FreeBSD: head/sys/arm/include/_limits.h 217145 2011-01-08 11:13:34Z tijl $
 */

#ifndef _MACHINE__LIMITS_H_
#define	_MACHINE__LIMITS_H_

/*
 * According to ANSI (section 2.2.4.2), the values below must be usable by
 * #if preprocessing directives.  Additionally, the expression must have the
 * same type as would an expression that is an object of the corresponding
 * type converted according to the integral promotions.  The subtraction for
 * INT_MIN, etc., is so the value is not unsigned; e.g., 0x80000000 is an
 * unsigned int for 32-bit two's complement ANSI compilers (section 3.1.3.2).
 */

#define	__CHAR_BIT	8		/* number of bits in a char */

#define	__SCHAR_MAX	0x7f		/* max value for a signed char */
#define	__SCHAR_MIN	(-0x7f - 1)	/* min value for a signed char */

#define	__UCHAR_MAX	0xff		/* max value for an unsigned char */

#define	__USHRT_MAX	0xffff		/* max value for an unsigned short */
#define	__SHRT_MAX	0x7fff		/* max value for a short */
#define	__SHRT_MIN	(-0x7fff - 1)	/* min value for a short */

#define	__UINT_MAX	0xffffffff	/* max value for an unsigned int */
#define	__INT_MAX	0x7fffffff	/* max value for an int */
#define	__INT_MIN	(-0x7fffffff - 1)	/* min value for an int */

#define	__ULONG_MAX	0xffffffffUL	/* max value for an unsigned long */
#define	__LONG_MAX	0x7fffffffL	/* max value for a long */
#define	__LONG_MIN	(-0x7fffffffL - 1)	/* min value for a long */

			/* max value for an unsigned long long */
#define	__ULLONG_MAX	0xffffffffffffffffULL
#define	__LLONG_MAX	0x7fffffffffffffffLL	/* max value for a long long */
#define	__LLONG_MIN	(-0x7fffffffffffffffLL - 1)  /* min for a long long */

#define	__SSIZE_MAX	__INT_MAX	/* max value for a ssize_t */

#define	__SIZE_T_MAX	__UINT_MAX	/* max value for a size_t */

#define	__OFF_MAX	__LLONG_MAX	/* max value for a off_t */
#define	__OFF_MIN	__LLONG_MIN	/* min value for a off_t */

/* Quads and long longs are the same size.  Ensure they stay in sync. */
#define	__UQUAD_MAX	__ULLONG_MAX	/* max value for a uquad_t */
#define	__QUAD_MAX	__LLONG_MAX	/* max value for a quad_t */
#define	__QUAD_MIN	__LLONG_MIN	/* min value for a quad_t */

#define	__LONG_BIT	32
#define	__WORD_BIT	32

/* Minimum signal stack size. */
#define	__MINSIGSTKSZ	(1024 * 4)

#endif /* !_MACHINE__LIMITS_H_ */
# 88 "/usr/include/machine/_limits.h" 3 4
# 23 "/usr/include/math.h" 2 3 4

/*
 * ANSI/POSIX
 */
extern const union __infinity_un {
	unsigned char	__uc[8];
	double		__ud;
} __infinity;

extern const union __nan_un {
	unsigned char	__uc[sizeof(float)];
	float		__uf;
} __nan;

#if __GNUC_PREREQ__(3, 3) || (defined(__INTEL_COMPILER) && __INTEL_COMPILER >= 800)
#define	__MATH_BUILTIN_CONSTANTS
#endif
# 40 "/usr/include/math.h" 3 4

#if __GNUC_PREREQ__(3, 0) && !defined(__INTEL_COMPILER)
#define	__MATH_BUILTIN_RELOPS
#endif
# 44 "/usr/include/math.h" 3 4

#ifdef __MATH_BUILTIN_CONSTANTS
#define	HUGE_VAL	__builtin_huge_val()
#else
# 48 "/usr/include/math.h" 3 4
#define	HUGE_VAL	(__infinity.__ud)
#endif
# 50 "/usr/include/math.h" 3 4

#if __ISO_C_VISIBLE >= 1999
#define	FP_ILOGB0	(-__INT_MAX)
#define	FP_ILOGBNAN	__INT_MAX

#ifdef __MATH_BUILTIN_CONSTANTS
#define	HUGE_VALF	__builtin_huge_valf()
#define	HUGE_VALL	__builtin_huge_vall()
#define	INFINITY	__builtin_inff()
#define	NAN		__builtin_nanf("")
#else
# 61 "/usr/include/math.h" 3 4
#define	HUGE_VALF	(float)HUGE_VAL
#define	HUGE_VALL	(long double)HUGE_VAL
#define	INFINITY	HUGE_VALF
#define	NAN		(__nan.__uf)
#endif /* __MATH_BUILTIN_CONSTANTS */
# 66 "/usr/include/math.h" 3 4

#define	MATH_ERRNO	1
#define	MATH_ERREXCEPT	2
#define	math_errhandling	MATH_ERREXCEPT

#define	FP_FAST_FMAF	1

/* Symbolic constants to classify floating point numbers. */
#define	FP_INFINITE	0x01
#define	FP_NAN		0x02
#define	FP_NORMAL	0x04
#define	FP_SUBNORMAL	0x08
#define	FP_ZERO		0x10

#if (__STDC_VERSION__ >= 201112L && defined(__clang__)) || \
    __has_extension(c_generic_selections)
#define	__fp_type_select(x, f, d, ld) _Generic((x),			\
    float: f(x),							\
    double: d(x),							\
    long double: ld(x),							\
    volatile float: f(x),						\
    volatile double: d(x),						\
    volatile long double: ld(x),					\
    volatile const float: f(x),						\
    volatile const double: d(x),					\
    volatile const long double: ld(x),					\
    const float: f(x),							\
    const double: d(x),							\
    const long double: ld(x))
#elif __GNUC_PREREQ__(3, 1) && !defined(__cplusplus)
# 96 "/usr/include/math.h" 3 4
#define	__fp_type_select(x, f, d, ld) __builtin_choose_expr(		\
    __builtin_types_compatible_p(__typeof(x), long double), ld(x),	\
    __builtin_choose_expr(						\
    __builtin_types_compatible_p(__typeof(x), double), d(x),		\
    __builtin_choose_expr(						\
    __builtin_types_compatible_p(__typeof(x), float), f(x), (void)0)))
#else
# 103 "/usr/include/math.h" 3 4
#define	 __fp_type_select(x, f, d, ld)					\
    ((sizeof(x) == sizeof(float)) ? f(x)				\
    : (sizeof(x) == sizeof(double)) ? d(x)				\
    : ld(x))
#endif
# 108 "/usr/include/math.h" 3 4

#define	fpclassify(x) \
	__fp_type_select(x, __fpclassifyf, __fpclassifyd, __fpclassifyl)
#define	isfinite(x) __fp_type_select(x, __isfinitef, __isfinite, __isfinitel)
#define	isinf(x) __fp_type_select(x, __isinff, __isinf, __isinfl)
#define	isnan(x) \
	__fp_type_select(x, __inline_isnanf, __inline_isnan, __inline_isnanl)
#define	isnormal(x) __fp_type_select(x, __isnormalf, __isnormal, __isnormall)

#ifdef __MATH_BUILTIN_RELOPS
#define	isgreater(x, y)		__builtin_isgreater((x), (y))
#define	isgreaterequal(x, y)	__builtin_isgreaterequal((x), (y))
#define	isless(x, y)		__builtin_isless((x), (y))
#define	islessequal(x, y)	__builtin_islessequal((x), (y))
#define	islessgreater(x, y)	__builtin_islessgreater((x), (y))
#define	isunordered(x, y)	__builtin_isunordered((x), (y))
#else
# 125 "/usr/include/math.h" 3 4
#define	isgreater(x, y)		(!isunordered((x), (y)) && (x) > (y))
#define	isgreaterequal(x, y)	(!isunordered((x), (y)) && (x) >= (y))
#define	isless(x, y)		(!isunordered((x), (y)) && (x) < (y))
#define	islessequal(x, y)	(!isunordered((x), (y)) && (x) <= (y))
#define	islessgreater(x, y)	(!isunordered((x), (y)) && \
					((x) > (y) || (y) > (x)))
#define	isunordered(x, y)	(isnan(x) || isnan(y))
#endif /* __MATH_BUILTIN_RELOPS */
# 133 "/usr/include/math.h" 3 4

#define	signbit(x) __fp_type_select(x, __signbitf, __signbit, __signbitl)

typedef	__double_t	double_t;
typedef	__float_t	float_t;
#endif /* __ISO_C_VISIBLE >= 1999 */
# 139 "/usr/include/math.h" 3 4

/*
 * XOPEN/SVID
 */
#if __BSD_VISIBLE || __XSI_VISIBLE
#define	M_E		2.7182818284590452354	/* e */
#define	M_LOG2E		1.4426950408889634074	/* log 2e */
#define	M_LOG10E	0.43429448190325182765	/* log 10e */
#define	M_LN2		0.69314718055994530942	/* log e2 */
#define	M_LN10		2.30258509299404568402	/* log e10 */
#define	M_PI		3.14159265358979323846	/* pi */
#define	M_PI_2		1.57079632679489661923	/* pi/2 */
#define	M_PI_4		0.78539816339744830962	/* pi/4 */
#define	M_1_PI		0.31830988618379067154	/* 1/pi */
#define	M_2_PI		0.63661977236758134308	/* 2/pi */
#define	M_2_SQRTPI	1.12837916709551257390	/* 2/sqrt(pi) */
#define	M_SQRT2		1.41421356237309504880	/* sqrt(2) */
#define	M_SQRT1_2	0.70710678118654752440	/* 1/sqrt(2) */

#define	MAXFLOAT	((float)3.40282346638528860e+38)
extern int signgam;
#endif /* __BSD_VISIBLE || __XSI_VISIBLE */
# 161 "/usr/include/math.h" 3 4

#if __BSD_VISIBLE
#if 0
/* Old value from 4.4BSD-Lite math.h; this is probably better. */
#define	HUGE		HUGE_VAL
#else
# 167 "/usr/include/math.h" 3 4
#define	HUGE		MAXFLOAT
#endif
# 169 "/usr/include/math.h" 3 4
#endif /* __BSD_VISIBLE */
# 170 "/usr/include/math.h" 3 4

/*
 * Most of these functions depend on the rounding mode and have the side
 * effect of raising floating-point exceptions, so they are not declared
 * as __pure2.  In C99, FENV_ACCESS affects the purity of these functions.
 */
__BEGIN_DECLS
/*
 * ANSI/POSIX
 */
int	__fpclassifyd(double) __pure2;
int	__fpclassifyf(float) __pure2;
int	__fpclassifyl(long double) __pure2;
int	__isfinitef(float) __pure2;
int	__isfinite(double) __pure2;
int	__isfinitel(long double) __pure2;
int	__isinff(float) __pure2;
int	__isinf(double) __pure2;
int	__isinfl(long double) __pure2;
int	__isnormalf(float) __pure2;
int	__isnormal(double) __pure2;
int	__isnormall(long double) __pure2;
int	__signbit(double) __pure2;
int	__signbitf(float) __pure2;
int	__signbitl(long double) __pure2;

static __inline int
__inline_isnan(__const double __x)
{

	return (__x != __x);
}

static __inline int
__inline_isnanf(__const float __x)
{

	return (__x != __x);
}

static __inline int
__inline_isnanl(__const long double __x)
{

	return (__x != __x);
}

/*
 * Version 2 of the Single UNIX Specification (UNIX98) defined isnan() and
 * isinf() as functions taking double.  C99, and the subsequent POSIX revisions
 * (SUSv3, POSIX.1-2001, define it as a macro that accepts any real floating
 * point type.  If we are targeting SUSv2 and C99 or C11 (or C++11) then we
 * expose the newer definition, assuming that the language spec takes
 * precedence over the operating system interface spec.
 */
#if	__XSI_VISIBLE > 0 && __XSI_VISIBLE < 600 && __ISO_C_VISIBLE < 1999
#undef isinf
#undef isnan
int	isinf(double);
int	isnan(double);
#endif
# 231 "/usr/include/math.h" 3 4

double	acos(double);
double	asin(double);
double	atan(double);
double	atan2(double, double);
double	cos(double);
double	sin(double);
double	tan(double);

double	cosh(double);
double	sinh(double);
double	tanh(double);

double	exp(double);
double	frexp(double, int *);	/* fundamentally !__pure2 */
double	ldexp(double, int);
double	log(double);
double	log10(double);
double	modf(double, double *);	/* fundamentally !__pure2 */

double	pow(double, double);
double	sqrt(double);

double	ceil(double);
double	fabs(double) __pure2;
double	floor(double);
double	fmod(double, double);

/*
 * These functions are not in C90.
 */
#if __BSD_VISIBLE || __ISO_C_VISIBLE >= 1999 || __XSI_VISIBLE
double	acosh(double);
double	asinh(double);
double	atanh(double);
double	cbrt(double);
double	erf(double);
double	erfc(double);
double	exp2(double);
double	expm1(double);
double	fma(double, double, double);
double	hypot(double, double);
int	ilogb(double) __pure2;
double	lgamma(double);
long long llrint(double);
long long llround(double);
double	log1p(double);
double	log2(double);
double	logb(double);
long	lrint(double);
long	lround(double);
double	nan(const char *) __pure2;
double	nextafter(double, double);
double	remainder(double, double);
double	remquo(double, double, int *);
double	rint(double);
#endif /* __BSD_VISIBLE || __ISO_C_VISIBLE >= 1999 || __XSI_VISIBLE */
# 288 "/usr/include/math.h" 3 4

#if __BSD_VISIBLE || __XSI_VISIBLE
double	j0(double);
double	j1(double);
double	jn(int, double);
double	y0(double);
double	y1(double);
double	yn(int, double);

#if __XSI_VISIBLE <= 500 || __BSD_VISIBLE
double	gamma(double);
#endif
# 300 "/usr/include/math.h" 3 4

#if __XSI_VISIBLE <= 600 || __BSD_VISIBLE
double	scalb(double, double);
#endif
# 304 "/usr/include/math.h" 3 4
#endif /* __BSD_VISIBLE || __XSI_VISIBLE */
# 305 "/usr/include/math.h" 3 4

#if __BSD_VISIBLE || __ISO_C_VISIBLE >= 1999
double	copysign(double, double) __pure2;
double	fdim(double, double);
double	fmax(double, double) __pure2;
double	fmin(double, double) __pure2;
double	nearbyint(double);
double	round(double);
double	scalbln(double, long);
double	scalbn(double, int);
double	tgamma(double);
double	trunc(double);
#endif
# 318 "/usr/include/math.h" 3 4

/*
 * BSD math library entry points
 */
#if __BSD_VISIBLE
double	drem(double, double);
int	finite(double) __pure2;
int	isnanf(float) __pure2;

/*
 * Reentrant version of gamma & lgamma; passes signgam back by reference
 * as the second argument; user must allocate space for signgam.
 */
double	gamma_r(double, int *);
double	lgamma_r(double, int *);

/*
 * IEEE Test Vector
 */
double	significand(double);
#endif /* __BSD_VISIBLE */
# 339 "/usr/include/math.h" 3 4

/* float versions of ANSI/POSIX functions */
#if __ISO_C_VISIBLE >= 1999
float	acosf(float);
float	asinf(float);
float	atanf(float);
float	atan2f(float, float);
float	cosf(float);
float	sinf(float);
float	tanf(float);

float	coshf(float);
float	sinhf(float);
float	tanhf(float);

float	exp2f(float);
float	expf(float);
float	expm1f(float);
float	frexpf(float, int *);	/* fundamentally !__pure2 */
int	ilogbf(float) __pure2;
float	ldexpf(float, int);
float	log10f(float);
float	log1pf(float);
float	log2f(float);
float	logf(float);
float	modff(float, float *);	/* fundamentally !__pure2 */

float	powf(float, float);
float	sqrtf(float);

float	ceilf(float);
float	fabsf(float) __pure2;
float	floorf(float);
float	fmodf(float, float);
float	roundf(float);

float	erff(float);
float	erfcf(float);
float	hypotf(float, float);
float	lgammaf(float);
float	tgammaf(float);

float	acoshf(float);
float	asinhf(float);
float	atanhf(float);
float	cbrtf(float);
float	logbf(float);
float	copysignf(float, float) __pure2;
long long llrintf(float);
long long llroundf(float);
long	lrintf(float);
long	lroundf(float);
float	nanf(const char *) __pure2;
float	nearbyintf(float);
float	nextafterf(float, float);
float	remainderf(float, float);
float	remquof(float, float, int *);
float	rintf(float);
float	scalblnf(float, long);
float	scalbnf(float, int);
float	truncf(float);

float	fdimf(float, float);
float	fmaf(float, float, float);
float	fmaxf(float, float) __pure2;
float	fminf(float, float) __pure2;
#endif
# 406 "/usr/include/math.h" 3 4

/*
 * float versions of BSD math library entry points
 */
#if __BSD_VISIBLE
float	dremf(float, float);
int	finitef(float) __pure2;
float	gammaf(float);
float	j0f(float);
float	j1f(float);
float	jnf(int, float);
float	scalbf(float, float);
float	y0f(float);
float	y1f(float);
float	ynf(int, float);

/*
 * Float versions of reentrant version of gamma & lgamma; passes
 * signgam back by reference as the second argument; user must
 * allocate space for signgam.
 */
float	gammaf_r(float, int *);
float	lgammaf_r(float, int *);

/*
 * float version of IEEE Test Vector
 */
float	significandf(float);
#endif	/* __BSD_VISIBLE */
# 435 "/usr/include/math.h" 3 4

/*
 * long double versions of ISO/POSIX math functions
 */
#if __ISO_C_VISIBLE >= 1999
long double	acoshl(long double);
long double	acosl(long double);
long double	asinhl(long double);
long double	asinl(long double);
long double	atan2l(long double, long double);
long double	atanhl(long double);
long double	atanl(long double);
long double	cbrtl(long double);
long double	ceill(long double);
long double	copysignl(long double, long double) __pure2;
long double	coshl(long double);
long double	cosl(long double);
long double	erfcl(long double);
long double	erfl(long double);
long double	exp2l(long double);
long double	expl(long double);
long double	expm1l(long double);
long double	fabsl(long double) __pure2;
long double	fdiml(long double, long double);
long double	floorl(long double);
long double	fmal(long double, long double, long double);
long double	fmaxl(long double, long double) __pure2;
long double	fminl(long double, long double) __pure2;
long double	fmodl(long double, long double);
long double	frexpl(long double value, int *); /* fundamentally !__pure2 */
long double	hypotl(long double, long double);
int		ilogbl(long double) __pure2;
long double	ldexpl(long double, int);
long double	lgammal(long double);
long long	llrintl(long double);
long long	llroundl(long double);
long double	log10l(long double);
long double	log1pl(long double);
long double	log2l(long double);
long double	logbl(long double);
long double	logl(long double);
long		lrintl(long double);
long		lroundl(long double);
long double	modfl(long double, long double *); /* fundamentally !__pure2 */
long double	nanl(const char *) __pure2;
long double	nearbyintl(long double);
long double	nextafterl(long double, long double);
double		nexttoward(double, long double);
float		nexttowardf(float, long double);
long double	nexttowardl(long double, long double);
long double	powl(long double, long double);
long double	remainderl(long double, long double);
long double	remquol(long double, long double, int *);
long double	rintl(long double);
long double	roundl(long double);
long double	scalblnl(long double, long);
long double	scalbnl(long double, int);
long double	sinhl(long double);
long double	sinl(long double);
long double	sqrtl(long double);
long double	tanhl(long double);
long double	tanl(long double);
long double	tgammal(long double);
long double	truncl(long double);
#endif /* __ISO_C_VISIBLE >= 1999 */
# 500 "/usr/include/math.h" 3 4

#if __BSD_VISIBLE
long double	lgammal_r(long double, int *);
#endif
# 504 "/usr/include/math.h" 3 4

__END_DECLS

#endif /* !_MATH_H_ */
# 508 "/usr/include/math.h" 3 4
# 30 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <limits.h>
#endif /* expanded by -frewrite-includes */
# 30 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/limits.h" 1 3 4
/*-
 * Copyright (c) 1988, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)limits.h	8.2 (Berkeley) 1/4/94
 * $FreeBSD: head/include/limits.h 293491 2016-01-09 14:53:23Z kevlo $
 */

#ifndef _LIMITS_H_
#define	_LIMITS_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/limits.h" 3 4
# 37 "/usr/include/limits.h" 3 4

#if __POSIX_VISIBLE
#define	_POSIX_ARG_MAX		4096
#define	_POSIX_LINK_MAX		8
#define	_POSIX_MAX_CANON	255
#define	_POSIX_MAX_INPUT	255
#define	_POSIX_NAME_MAX		14
#define	_POSIX_PIPE_BUF		512
#define	_POSIX_SSIZE_MAX	32767
#define	_POSIX_STREAM_MAX	8

#if __POSIX_VISIBLE >= 200112
#define	_POSIX_CHILD_MAX	25
#define	_POSIX_NGROUPS_MAX	8
#define	_POSIX_OPEN_MAX		20
#define	_POSIX_PATH_MAX		256
#define	_POSIX_TZNAME_MAX	6
#else
# 55 "/usr/include/limits.h" 3 4
#define	_POSIX_CHILD_MAX	6
#define	_POSIX_NGROUPS_MAX	0
#define	_POSIX_OPEN_MAX		16
#define	_POSIX_PATH_MAX		255
#define	_POSIX_TZNAME_MAX	3
#endif
# 61 "/usr/include/limits.h" 3 4

#if __POSIX_VISIBLE >= 200112
#define	BC_BASE_MAX		   99	/* max ibase/obase values in bc(1) */
#define	BC_DIM_MAX		 2048	/* max array elements in bc(1) */
#define	BC_SCALE_MAX		   99	/* max scale value in bc(1) */
#define	BC_STRING_MAX		 1000	/* max const string length in bc(1) */
#define	CHARCLASS_NAME_MAX	   14	/* max character class name size */
#define	COLL_WEIGHTS_MAX	   10	/* max weights for order keyword */
#define	EXPR_NEST_MAX		   32	/* max expressions nested in expr(1) */
#define	LINE_MAX		 2048	/* max bytes in an input line */
#define	RE_DUP_MAX		  255	/* max RE's in interval notation */

#define	_POSIX2_BC_BASE_MAX	99
#define	_POSIX2_BC_DIM_MAX	2048
#define	_POSIX2_BC_SCALE_MAX	99
#define	_POSIX2_BC_STRING_MAX	1000
#define	_POSIX2_CHARCLASS_NAME_MAX 14
#define	_POSIX2_COLL_WEIGHTS_MAX 2
#define	_POSIX2_EQUIV_CLASS_MAX	2
#define	_POSIX2_EXPR_NEST_MAX	32
#define	_POSIX2_LINE_MAX	2048
#define	_POSIX2_RE_DUP_MAX	255
#endif
# 84 "/usr/include/limits.h" 3 4
#endif
# 85 "/usr/include/limits.h" 3 4

#if __POSIX_VISIBLE >= 199309
#define	_POSIX_AIO_LISTIO_MAX	2
#define	_POSIX_AIO_MAX		1
#define	_POSIX_DELAYTIMER_MAX	32
#define	_POSIX_MQ_OPEN_MAX	8
#define	_POSIX_MQ_PRIO_MAX	32
#define	_POSIX_RTSIG_MAX	8
#define	_POSIX_SEM_NSEMS_MAX	256
#define	_POSIX_SEM_VALUE_MAX	32767
#define	_POSIX_SIGQUEUE_MAX	32
#define	_POSIX_TIMER_MAX	32

#define	_POSIX_CLOCKRES_MIN	20000000
#endif
# 100 "/usr/include/limits.h" 3 4

#if __POSIX_VISIBLE >= 199506
#define	_POSIX_THREAD_DESTRUCTOR_ITERATIONS 4
#define	_POSIX_THREAD_KEYS_MAX	128
#define	_POSIX_THREAD_THREADS_MAX 64
#endif
# 106 "/usr/include/limits.h" 3 4

#if __POSIX_VISIBLE >= 200112
#define	_POSIX_HOST_NAME_MAX	255
#define	_POSIX_LOGIN_NAME_MAX	9
#define	_POSIX_SS_REPL_MAX	4
#define	_POSIX_SYMLINK_MAX	255
#define	_POSIX_SYMLOOP_MAX	8
#define	_POSIX_TRACE_EVENT_NAME_MAX 30
#define	_POSIX_TRACE_NAME_MAX	8
#define	_POSIX_TRACE_SYS_MAX	8
#define	_POSIX_TRACE_USER_EVENT_MAX 32
#define	_POSIX_TTY_NAME_MAX	9

#define	_POSIX_RE_DUP_MAX	_POSIX2_RE_DUP_MAX
#endif
# 121 "/usr/include/limits.h" 3 4

#if __XSI_VISIBLE || __POSIX_VISIBLE >= 200809
#define	NL_ARGMAX		99	/* max # of position args for printf */
#define	NL_MSGMAX		32767
#define	NL_SETMAX		255
#define	NL_TEXTMAX		2048
#endif
# 128 "/usr/include/limits.h" 3 4

#if __XSI_VISIBLE
#define	_XOPEN_IOV_MAX		16
#define	_XOPEN_NAME_MAX		255
#define	_XOPEN_PATH_MAX		1024
#define	PASS_MAX		128	/* _PASSWORD_LEN from <pwd.h> */

#define	NL_LANGMAX		31	/* max LANG name length */
#define	NL_NMAX			1
#endif
# 138 "/usr/include/limits.h" 3 4

#define	MB_LEN_MAX		6	/* 31-bit UTF-8 */

#if 0 /* expanded by -frewrite-includes */
#include <sys/limits.h>
#endif /* expanded by -frewrite-includes */
# 141 "/usr/include/limits.h" 3 4
# 1 "/usr/include/sys/limits.h" 1 3 4
/*-
 * Copyright (c) 1988, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/limits.h 219271 2011-03-04 19:28:27Z jilles $
 */

#ifndef _SYS_LIMITS_H_
#define	_SYS_LIMITS_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 35 "/usr/include/sys/limits.h" 3 4
# 36 "/usr/include/sys/limits.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/_limits.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/limits.h" 3 4
# 37 "/usr/include/sys/limits.h" 3 4

#define	CHAR_BIT	__CHAR_BIT	/* number of bits in a char */

#define	SCHAR_MAX	__SCHAR_MAX	/* max value for a signed char */
#define	SCHAR_MIN	__SCHAR_MIN	/* min value for a signed char */

#define	UCHAR_MAX	__UCHAR_MAX	/* max value for an unsigned char */

#ifdef __CHAR_UNSIGNED__
#define	CHAR_MAX	UCHAR_MAX	/* max value for a char */
#define	CHAR_MIN	0		/* min value for a char */
#else
# 49 "/usr/include/sys/limits.h" 3 4
#define	CHAR_MAX	SCHAR_MAX
#define	CHAR_MIN	SCHAR_MIN
#endif
# 52 "/usr/include/sys/limits.h" 3 4

#define	USHRT_MAX	__USHRT_MAX	/* max value for an unsigned short */
#define	SHRT_MAX	__SHRT_MAX	/* max value for a short */
#define	SHRT_MIN	__SHRT_MIN	/* min value for a short */

#define	UINT_MAX	__UINT_MAX	/* max value for an unsigned int */
#define	INT_MAX		__INT_MAX	/* max value for an int */
#define	INT_MIN		__INT_MIN	/* min value for an int */

#define	ULONG_MAX	__ULONG_MAX	/* max for an unsigned long */
#define	LONG_MAX	__LONG_MAX	/* max for a long */
#define	LONG_MIN	__LONG_MIN	/* min for a long */

#ifdef __LONG_LONG_SUPPORTED
#define	ULLONG_MAX	__ULLONG_MAX	/* max for an unsigned long long */
#define	LLONG_MAX	__LLONG_MAX	/* max for a long long */
#define	LLONG_MIN	__LLONG_MIN	/* min for a long long */
#endif
# 70 "/usr/include/sys/limits.h" 3 4

#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SSIZE_MAX	__SSIZE_MAX	/* max value for an ssize_t */
#endif
# 74 "/usr/include/sys/limits.h" 3 4

#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#define	SIZE_T_MAX	__SIZE_T_MAX	/* max value for a size_t */

#define	OFF_MAX		__OFF_MAX	/* max value for an off_t */
#define	OFF_MIN		__OFF_MIN	/* min value for an off_t */
#endif
# 81 "/usr/include/sys/limits.h" 3 4

#if __BSD_VISIBLE
#define	GID_MAX		UINT_MAX	/* max value for a gid_t */
#define	UID_MAX		UINT_MAX	/* max value for a uid_t */

#define	UQUAD_MAX	(__UQUAD_MAX)	/* max value for a uquad_t */
#define	QUAD_MAX	(__QUAD_MAX)	/* max value for a quad_t */
#define	QUAD_MIN	(__QUAD_MIN)	/* min value for a quad_t */
#endif
# 90 "/usr/include/sys/limits.h" 3 4

#if __XSI_VISIBLE || __POSIX_VISIBLE >= 200809
#define	LONG_BIT	__LONG_BIT
#define	WORD_BIT	__WORD_BIT
#endif
# 95 "/usr/include/sys/limits.h" 3 4

#if __POSIX_VISIBLE
#define	MQ_PRIO_MAX	64
#endif
# 99 "/usr/include/sys/limits.h" 3 4

#endif /* !_SYS_LIMITS_H_ */
# 101 "/usr/include/sys/limits.h" 3 4
# 142 "/usr/include/limits.h" 2 3 4

#if __POSIX_VISIBLE
#if 0 /* expanded by -frewrite-includes */
#include <sys/syslimits.h>
#endif /* expanded by -frewrite-includes */
# 144 "/usr/include/limits.h" 3 4
# 1 "/usr/include/sys/syslimits.h" 1 3 4
/*-
 * Copyright (c) 1988, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)syslimits.h	8.1 (Berkeley) 6/2/93
 * $FreeBSD: head/sys/sys/syslimits.h 194498 2009-06-19 17:10:35Z brooks $
 */

#ifndef _SYS_SYSLIMITS_H_
#define _SYS_SYSLIMITS_H_

#if !defined(_KERNEL) && !defined(_LIMITS_H_) && !defined(_SYS_PARAM_H_)
#ifndef _SYS_CDEFS_H_
#error this file needs sys/cdefs.h as a prerequisite
#endif
# 40 "/usr/include/sys/syslimits.h" 3 4
#ifdef __CC_SUPPORTS_WARNING
#warning "No user-serviceable parts inside."
#endif
# 43 "/usr/include/sys/syslimits.h" 3 4
#endif
# 44 "/usr/include/sys/syslimits.h" 3 4

/*
 * Do not add any new variables here.  (See the comment at the end of
 * the file for why.)
 */
#define	ARG_MAX			262144	/* max bytes for an exec function */
#ifndef CHILD_MAX
#define	CHILD_MAX		   40	/* max simultaneous processes */
#endif
# 53 "/usr/include/sys/syslimits.h" 3 4
#define	LINK_MAX		32767	/* max file link count */
#define	MAX_CANON		  255	/* max bytes in term canon input line */
#define	MAX_INPUT		  255	/* max bytes in terminal input */
#define	NAME_MAX		  255	/* max bytes in a file name */
#ifndef NGROUPS_MAX
#define	NGROUPS_MAX	 	 1023	/* max supplemental group id's */
#endif
# 60 "/usr/include/sys/syslimits.h" 3 4
#ifndef OPEN_MAX
#define	OPEN_MAX		   64	/* max open files per process */
#endif
# 63 "/usr/include/sys/syslimits.h" 3 4
#define	PATH_MAX		 1024	/* max bytes in pathname */
#define	PIPE_BUF		  512	/* max bytes for atomic pipe writes */
#define	IOV_MAX			 1024	/* max elements in i/o vector */

/*
 * We leave the following values undefined to force applications to either
 * assume conservative values or call sysconf() to get the current value.
 *
 * HOST_NAME_MAX
 *
 * (We should do this for most of the values currently defined here,
 * but many programs are not prepared to deal with this yet.)
 */
#endif
# 77 "/usr/include/sys/syslimits.h" 3 4
# 145 "/usr/include/limits.h" 2 3 4
#endif
# 146 "/usr/include/limits.h" 3 4

#endif /* !_LIMITS_H_ */
# 148 "/usr/include/limits.h" 3 4
# 31 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <stdlib.h>
#endif /* expanded by -frewrite-includes */
# 31 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/stdlib.h" 1 3 4
/*-
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)stdlib.h	8.5 (Berkeley) 5/19/95
 * $FreeBSD: head/include/stdlib.h 297212 2016-03-23 19:17:12Z dim $
 */

#ifndef _STDLIB_H_
#define	_STDLIB_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/stdlib.h" 3 4
# 37 "/usr/include/stdlib.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/stdlib.h" 3 4
# 38 "/usr/include/stdlib.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/stdlib.h" 3 4
# 39 "/usr/include/stdlib.h" 3 4

#if __BSD_VISIBLE
#ifndef _RUNE_T_DECLARED
typedef	__rune_t	rune_t;
#define	_RUNE_T_DECLARED
#endif
# 45 "/usr/include/stdlib.h" 3 4
#endif
# 46 "/usr/include/stdlib.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 51 "/usr/include/stdlib.h" 3 4

#ifndef	__cplusplus
#ifndef _WCHAR_T_DECLARED
typedef	___wchar_t	wchar_t;
#define	_WCHAR_T_DECLARED
#endif
# 57 "/usr/include/stdlib.h" 3 4
#endif
# 58 "/usr/include/stdlib.h" 3 4

typedef struct {
	int	quot;		/* quotient */
	int	rem;		/* remainder */
} div_t;

typedef struct {
	long	quot;
	long	rem;
} ldiv_t;

#define	EXIT_FAILURE	1
#define	EXIT_SUCCESS	0

#define	RAND_MAX	0x7ffffffd

__BEGIN_DECLS
#ifdef _XLOCALE_H_
#if 0 /* expanded by -frewrite-includes */
#include <xlocale/_stdlib.h>
#endif /* expanded by -frewrite-includes */
# 76 "/usr/include/stdlib.h" 3 4
# 77 "/usr/include/stdlib.h" 3 4
#endif
# 78 "/usr/include/stdlib.h" 3 4
extern int __mb_cur_max;
extern int ___mb_cur_max(void);
#define	MB_CUR_MAX	(___mb_cur_max())

_Noreturn void	 abort(void);
int	 abs(int) __pure2;
int	 atexit(void (*)(void));
double	 atof(const char *);
int	 atoi(const char *);
long	 atol(const char *);
void	*bsearch(const void *, const void *, size_t,
	    size_t, int (*)(const void *, const void *));
void	*calloc(size_t, size_t) __malloc_like __result_use_check
	     __alloc_size(1) __alloc_size(2);
div_t	 div(int, int) __pure2;
_Noreturn void	 exit(int);
void	 free(void *);
char	*getenv(const char *);
long	 labs(long) __pure2;
ldiv_t	 ldiv(long, long) __pure2;
void	*malloc(size_t) __malloc_like __result_use_check __alloc_size(1);
int	 mblen(const char *, size_t);
size_t	 mbstowcs(wchar_t * __restrict , const char * __restrict, size_t);
int	 mbtowc(wchar_t * __restrict, const char * __restrict, size_t);
void	 qsort(void *, size_t, size_t,
	    int (*)(const void *, const void *));
int	 rand(void);
void	*realloc(void *, size_t) __result_use_check __alloc_size(2);
void	 srand(unsigned);
double	 strtod(const char * __restrict, char ** __restrict);
float	 strtof(const char * __restrict, char ** __restrict);
long	 strtol(const char * __restrict, char ** __restrict, int);
long double
	 strtold(const char * __restrict, char ** __restrict);
unsigned long
	 strtoul(const char * __restrict, char ** __restrict, int);
int	 system(const char *);
int	 wctomb(char *, wchar_t);
size_t	 wcstombs(char * __restrict, const wchar_t * __restrict, size_t);

/*
 * Functions added in C99 which we make conditionally available in the
 * BSD^C89 namespace if the compiler supports `long long'.
 * The #if test is more complicated than it ought to be because
 * __BSD_VISIBLE implies __ISO_C_VISIBLE == 1999 *even if* `long long'
 * is not supported in the compilation environment (which therefore means
 * that it can't really be ISO C99).
 *
 * (The only other extension made by C99 in thie header is _Exit().)
 */
#if __ISO_C_VISIBLE >= 1999 || defined(__cplusplus)
#ifdef __LONG_LONG_SUPPORTED
/* LONGLONG */
typedef struct {
	long long quot;
	long long rem;
} lldiv_t;

/* LONGLONG */
long long
	 atoll(const char *);
/* LONGLONG */
long long
	 llabs(long long) __pure2;
/* LONGLONG */
lldiv_t	 lldiv(long long, long long) __pure2;
/* LONGLONG */
long long
	 strtoll(const char * __restrict, char ** __restrict, int);
/* LONGLONG */
unsigned long long
	 strtoull(const char * __restrict, char ** __restrict, int);
#endif /* __LONG_LONG_SUPPORTED */
# 151 "/usr/include/stdlib.h" 3 4

_Noreturn void	 _Exit(int);
#endif /* __ISO_C_VISIBLE >= 1999 */
# 154 "/usr/include/stdlib.h" 3 4

/*
 * If we're in a mode greater than C99, expose C11 functions.
 */
#if __ISO_C_VISIBLE >= 2011 || __cplusplus >= 201103L
void *	aligned_alloc(size_t, size_t) __malloc_like __alloc_align(1)
	    __alloc_size(2);
int	at_quick_exit(void (*)(void));
_Noreturn void
	quick_exit(int);
#endif /* __ISO_C_VISIBLE >= 2011 */
# 165 "/usr/include/stdlib.h" 3 4
/*
 * Extensions made by POSIX relative to C.
 */
#if __POSIX_VISIBLE >= 199506 || __XSI_VISIBLE
char	*realpath(const char * __restrict, char * __restrict);
#endif
# 171 "/usr/include/stdlib.h" 3 4
#if __POSIX_VISIBLE >= 199506
int	 rand_r(unsigned *);			/* (TSF) */
#endif
# 174 "/usr/include/stdlib.h" 3 4
#if __POSIX_VISIBLE >= 200112
int	 posix_memalign(void **, size_t, size_t) __nonnull(1) __alloc_align(2)
	    __alloc_size(3);			/* (ADV) */
int	 setenv(const char *, const char *, int);
int	 unsetenv(const char *);
#endif
# 180 "/usr/include/stdlib.h" 3 4

#if __POSIX_VISIBLE >= 200809 || __XSI_VISIBLE
int	 getsubopt(char **, char *const *, char **);
#ifndef _MKDTEMP_DECLARED
char	*mkdtemp(char *);
#define	_MKDTEMP_DECLARED
#endif
# 187 "/usr/include/stdlib.h" 3 4
#ifndef _MKSTEMP_DECLARED
int	 mkstemp(char *);
#define	_MKSTEMP_DECLARED
#endif
# 191 "/usr/include/stdlib.h" 3 4
#endif /* __POSIX_VISIBLE >= 200809 || __XSI_VISIBLE */
# 192 "/usr/include/stdlib.h" 3 4

/*
 * The only changes to the XSI namespace in revision 6 were the deletion
 * of the ttyslot() and valloc() functions, which FreeBSD never declared
 * in this header.  For revision 7, ecvt(), fcvt(), and gcvt(), which
 * FreeBSD also does not have, and mktemp(), are to be deleted.
 */
#if __XSI_VISIBLE
/* XXX XSI requires pollution from <sys/wait.h> here.  We'd rather not. */
long	 a64l(const char *);
double	 drand48(void);
/* char	*ecvt(double, int, int * __restrict, int * __restrict); */
double	 erand48(unsigned short[3]);
/* char	*fcvt(double, int, int * __restrict, int * __restrict); */
/* char	*gcvt(double, int, int * __restrict, int * __restrict); */
int	 grantpt(int);
char	*initstate(unsigned long /* XSI requires u_int */, char *, long);
long	 jrand48(unsigned short[3]);
char	*l64a(long);
void	 lcong48(unsigned short[7]);
long	 lrand48(void);
#if !defined(_MKTEMP_DECLARED) && (__BSD_VISIBLE || __XSI_VISIBLE <= 600)
char	*mktemp(char *);
#define	_MKTEMP_DECLARED
#endif
# 217 "/usr/include/stdlib.h" 3 4
long	 mrand48(void);
long	 nrand48(unsigned short[3]);
int	 posix_openpt(int);
char	*ptsname(int);
int	 putenv(char *);
long	 random(void);
unsigned short
	*seed48(unsigned short[3]);
#ifndef _SETKEY_DECLARED
int	 setkey(const char *);
#define	_SETKEY_DECLARED
#endif
# 229 "/usr/include/stdlib.h" 3 4
char	*setstate(/* const */ char *);
void	 srand48(long);
void	 srandom(unsigned long);
int	 unlockpt(int);
#endif /* __XSI_VISIBLE */
# 234 "/usr/include/stdlib.h" 3 4

#if __BSD_VISIBLE
extern const char *malloc_conf;
extern void (*malloc_message)(void *, const char *);

/*
 * The alloca() function can't be implemented in C, and on some
 * platforms it can't be implemented at all as a callable function.
 * The GNU C compiler provides a built-in alloca() which we can use;
 * in all other cases, provide a prototype, mainly to pacify various
 * incarnations of lint.  On platforms where alloca() is not in libc,
 * programs which use it will fail to link when compiled with non-GNU
 * compilers.
 */
#if __GNUC__ >= 2 || defined(__INTEL_COMPILER)
#undef  alloca	/* some GNU bits try to get cute and define this on their own */
#define alloca(sz) __builtin_alloca(sz)
#elif defined(lint)
# 252 "/usr/include/stdlib.h" 3 4
void	*alloca(size_t);
#endif
# 254 "/usr/include/stdlib.h" 3 4

void	 abort2(const char *, int, void **) __dead2;
__uint32_t
	 arc4random(void);
void	 arc4random_addrandom(unsigned char *, int);
void	 arc4random_buf(void *, size_t);
void	 arc4random_stir(void);
__uint32_t 
	 arc4random_uniform(__uint32_t);
#ifdef __BLOCKS__
int	 atexit_b(void (^)(void));
void	*bsearch_b(const void *, const void *, size_t,
	    size_t, int (^)(const void *, const void *));
#endif
# 268 "/usr/include/stdlib.h" 3 4
char	*getbsize(int *, long *);
					/* getcap(3) functions */
char	*cgetcap(char *, const char *, int);
int	 cgetclose(void);
int	 cgetent(char **, char **, const char *);
int	 cgetfirst(char **, char **);
int	 cgetmatch(const char *, const char *);
int	 cgetnext(char **, char **);
int	 cgetnum(char *, const char *, long *);
int	 cgetset(const char *);
int	 cgetstr(char *, const char *, char **);
int	 cgetustr(char *, const char *, char **);

int	 daemon(int, int);
char	*devname(__dev_t, __mode_t);
char	*devname_r(__dev_t, __mode_t, char *, int);
char	*fdevname(int);
char	*fdevname_r(int, char *, int);
int	 getloadavg(double [], int);
const char *
	 getprogname(void);

int	 heapsort(void *, size_t, size_t, int (*)(const void *, const void *));
#ifdef __BLOCKS__
int	 heapsort_b(void *, size_t, size_t, int (^)(const void *, const void *));
void	 qsort_b(void *, size_t, size_t,
	    int (^)(const void *, const void *));
#endif
# 296 "/usr/include/stdlib.h" 3 4
int	 l64a_r(long, char *, int);
int	 mergesort(void *, size_t, size_t, int (*)(const void *, const void *));
#ifdef __BLOCKS__
int	 mergesort_b(void *, size_t, size_t, int (^)(const void *, const void *));
#endif
# 301 "/usr/include/stdlib.h" 3 4
int	 mkostemp(char *, int);
int	 mkostemps(char *, int, int);
void	 qsort_r(void *, size_t, size_t, void *,
	    int (*)(void *, const void *, const void *));
int	 radixsort(const unsigned char **, int, const unsigned char *,
	    unsigned);
void	*reallocarray(void *, size_t, size_t) __result_use_check __alloc_size(2)
	    __alloc_size(3);
void	*reallocf(void *, size_t) __alloc_size(2);
int	 rpmatch(const char *);
void	 setprogname(const char *);
int	 sradixsort(const unsigned char **, int, const unsigned char *,
	    unsigned);
void	 sranddev(void);
void	 srandomdev(void);
long long
	strtonum(const char *, long long, long long, const char **);

/* Deprecated interfaces, to be removed in FreeBSD 6.0. */
__int64_t
	 strtoq(const char *, char **, int);
__uint64_t
	 strtouq(const char *, char **, int);

extern char *suboptarg;			/* getsubopt(3) external variable */
#endif /* __BSD_VISIBLE */
# 327 "/usr/include/stdlib.h" 3 4
__END_DECLS

#endif /* !_STDLIB_H_ */
# 330 "/usr/include/stdlib.h" 3 4
# 32 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <string.h>
#endif /* expanded by -frewrite-includes */
# 32 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/string.h" 1 3 4
/*-
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)string.h	8.1 (Berkeley) 6/2/93
 * $FreeBSD: head/include/string.h 265878 2014-05-11 13:48:21Z jilles $
 */

#ifndef _STRING_H_
#define	_STRING_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/string.h" 3 4
# 37 "/usr/include/string.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/string.h" 3 4
# 38 "/usr/include/string.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/string.h" 3 4
# 39 "/usr/include/string.h" 3 4

/*
 * Prototype functions which were historically defined in <string.h>, but
 * are required by POSIX to be prototyped in <strings.h>.
 */
#if __BSD_VISIBLE
#if 0 /* expanded by -frewrite-includes */
#include <strings.h>
#endif /* expanded by -frewrite-includes */
# 45 "/usr/include/string.h" 3 4
# 1 "/usr/include/strings.h" 1 3 4
/*-
 * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/include/strings.h 272673 2014-10-07 04:54:11Z delphij $
 */

#ifndef _STRINGS_H_
#define	_STRINGS_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 32 "/usr/include/strings.h" 3 4
# 33 "/usr/include/strings.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 33 "/usr/include/strings.h" 3 4
# 34 "/usr/include/strings.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 39 "/usr/include/strings.h" 3 4

__BEGIN_DECLS
#if __BSD_VISIBLE || __POSIX_VISIBLE <= 200112
int	 bcmp(const void *, const void *, size_t) __pure;	/* LEGACY */
void	 bcopy(const void *, void *, size_t);			/* LEGACY */
void	 bzero(void *, size_t);					/* LEGACY */
#endif
# 46 "/usr/include/strings.h" 3 4
#if __BSD_VISIBLE
void	 explicit_bzero(void *, size_t);
#endif
# 49 "/usr/include/strings.h" 3 4
#if __XSI_VISIBLE
int	 ffs(int) __pure2;
#endif
# 52 "/usr/include/strings.h" 3 4
#if __BSD_VISIBLE
int	 ffsl(long) __pure2;
int	 ffsll(long long) __pure2;
int	 fls(int) __pure2;
int	 flsl(long) __pure2;
int	 flsll(long long) __pure2;
#endif
# 59 "/usr/include/strings.h" 3 4
#if __BSD_VISIBLE || __POSIX_VISIBLE <= 200112
char	*index(const char *, int) __pure;			/* LEGACY */
char	*rindex(const char *, int) __pure;			/* LEGACY */
#endif
# 63 "/usr/include/strings.h" 3 4
int	 strcasecmp(const char *, const char *) __pure;
int	 strncasecmp(const char *, const char *, size_t) __pure;

#if __POSIX_VISIBLE >= 200809 || defined(_XLOCALE_H_)
#if 0 /* expanded by -frewrite-includes */
#include <xlocale/_strings.h>
#endif /* expanded by -frewrite-includes */
# 67 "/usr/include/strings.h" 3 4
# 1 "/usr/include/xlocale/_strings.h" 1 3 4
/*-
 * Copyright (c) 2011, 2012 The FreeBSD Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/include/xlocale/_strings.h 266865 2014-05-30 01:09:07Z pfg $
 */

#ifndef _LOCALE_T_DEFINED
#define _LOCALE_T_DEFINED
typedef struct	_xlocale *locale_t;
#endif
# 33 "/usr/include/xlocale/_strings.h" 3 4

/*
 * This file is included from both strings.h and xlocale.h.  We need to expose
 * the declarations unconditionally if we are included from xlocale.h, but only
 * if we are in POSIX2008 mode if included from string.h.
 */

#ifndef _XLOCALE_STRINGS1_H
#define _XLOCALE_STRINGS1_H

/*
 * POSIX2008 functions
 */
int	 strcasecmp_l(const char *, const char *, locale_t);
int	 strncasecmp_l(const char *, const char *, size_t, locale_t);
#endif /* _XLOCALE_STRINGS1_H */
# 49 "/usr/include/xlocale/_strings.h" 3 4
# 68 "/usr/include/strings.h" 2 3 4
#endif
# 69 "/usr/include/strings.h" 3 4
__END_DECLS

#endif /* _STRINGS_H_ */
# 72 "/usr/include/strings.h" 3 4
# 46 "/usr/include/string.h" 2 3 4
#endif
# 47 "/usr/include/string.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 52 "/usr/include/string.h" 3 4

__BEGIN_DECLS
#if __XSI_VISIBLE >= 600
void	*memccpy(void * __restrict, const void * __restrict, int, size_t);
#endif
# 57 "/usr/include/string.h" 3 4
void	*memchr(const void *, int, size_t) __pure;
#if __BSD_VISIBLE
void	*memrchr(const void *, int, size_t) __pure;
#endif
# 61 "/usr/include/string.h" 3 4
int	 memcmp(const void *, const void *, size_t) __pure;
void	*memcpy(void * __restrict, const void * __restrict, size_t);
#if __BSD_VISIBLE
void	*memmem(const void *, size_t, const void *, size_t) __pure;
#endif
# 66 "/usr/include/string.h" 3 4
void	*memmove(void *, const void *, size_t);
void	*memset(void *, int, size_t);
#if __POSIX_VISIBLE >= 200809
char	*stpcpy(char * __restrict, const char * __restrict);
char	*stpncpy(char * __restrict, const char * __restrict, size_t);
#endif
# 72 "/usr/include/string.h" 3 4
#if __BSD_VISIBLE
char	*strcasestr(const char *, const char *) __pure;
#endif
# 75 "/usr/include/string.h" 3 4
char	*strcat(char * __restrict, const char * __restrict);
char	*strchr(const char *, int) __pure;
#if __BSD_VISIBLE
char	*strchrnul(const char*, int) __pure;
#endif
# 80 "/usr/include/string.h" 3 4
int	 strcmp(const char *, const char *) __pure;
int	 strcoll(const char *, const char *);
char	*strcpy(char * __restrict, const char * __restrict);
size_t	 strcspn(const char *, const char *) __pure;
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
char	*strdup(const char *) __malloc_like;
#endif
# 87 "/usr/include/string.h" 3 4
char	*strerror(int);
#if __POSIX_VISIBLE >= 200112
int	 strerror_r(int, char *, size_t);
#endif
# 91 "/usr/include/string.h" 3 4
#if __BSD_VISIBLE
size_t	 strlcat(char * __restrict, const char * __restrict, size_t);
size_t	 strlcpy(char * __restrict, const char * __restrict, size_t);
#endif
# 95 "/usr/include/string.h" 3 4
size_t	 strlen(const char *) __pure;
#if __BSD_VISIBLE
void	 strmode(int, char *);
#endif
# 99 "/usr/include/string.h" 3 4
char	*strncat(char * __restrict, const char * __restrict, size_t);
int	 strncmp(const char *, const char *, size_t) __pure;
char	*strncpy(char * __restrict, const char * __restrict, size_t);
#if __POSIX_VISIBLE >= 200809
char	*strndup(const char *, size_t) __malloc_like;
size_t	 strnlen(const char *, size_t) __pure;
#endif
# 106 "/usr/include/string.h" 3 4
#if __BSD_VISIBLE
char	*strnstr(const char *, const char *, size_t) __pure;
#endif
# 109 "/usr/include/string.h" 3 4
char	*strpbrk(const char *, const char *) __pure;
char	*strrchr(const char *, int) __pure;
#if __BSD_VISIBLE
char	*strsep(char **, const char *);
#endif
# 114 "/usr/include/string.h" 3 4
#if __POSIX_VISIBLE >= 200809
char	*strsignal(int);
#endif
# 117 "/usr/include/string.h" 3 4
size_t	 strspn(const char *, const char *) __pure;
char	*strstr(const char *, const char *) __pure;
char	*strtok(char * __restrict, const char * __restrict);
#if __POSIX_VISIBLE >= 199506 || __XSI_VISIBLE >= 500
char	*strtok_r(char *, const char *, char **);
#endif
# 123 "/usr/include/string.h" 3 4
size_t	 strxfrm(char * __restrict, const char * __restrict, size_t);
#if __BSD_VISIBLE

#ifndef _SWAB_DECLARED
#define _SWAB_DECLARED

#ifndef _SSIZE_T_DECLARED
typedef	__ssize_t	ssize_t;
#define	_SSIZE_T_DECLARED
#endif /* _SIZE_T_DECLARED */
# 133 "/usr/include/string.h" 3 4

void	 swab(const void * __restrict, void * __restrict, ssize_t);
#endif /* _SWAB_DECLARED */
# 136 "/usr/include/string.h" 3 4

#endif /* __BSD_VISIBLE */
# 138 "/usr/include/string.h" 3 4

#if __POSIX_VISIBLE >= 200809 || defined(_XLOCALE_H_)
#if 0 /* expanded by -frewrite-includes */
#include <xlocale/_string.h>
#endif /* expanded by -frewrite-includes */
# 140 "/usr/include/string.h" 3 4
# 1 "/usr/include/xlocale/_string.h" 1 3 4
/*-
 * Copyright (c) 2011, 2012 The FreeBSD Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/include/xlocale/_string.h 266865 2014-05-30 01:09:07Z pfg $
 */

#ifndef _LOCALE_T_DEFINED
#define _LOCALE_T_DEFINED
typedef struct	_xlocale *locale_t;
#endif
# 33 "/usr/include/xlocale/_string.h" 3 4

/*
 * This file is included from both string.h and xlocale.h.  We need to expose
 * the declarations unconditionally if we are included from xlocale.h, but only
 * if we are in POSIX2008 mode if included from string.h.
 */

#ifndef _XLOCALE_STRING1_H
#define _XLOCALE_STRING1_H

/*
 * POSIX2008 functions
 */
int	 strcoll_l(const char *, const char *, locale_t);
size_t	 strxfrm_l(char *, const char *, size_t, locale_t);
#endif /* _XLOCALE_STRING1_H */
# 49 "/usr/include/xlocale/_string.h" 3 4

/*
 * xlocale extensions
 */
#ifdef _XLOCALE_H_
#ifndef _XLOCALE_STRING2_H
#define _XLOCALE_STRING2_H
char	*strcasestr_l(const char *, const char *, locale_t);

#endif /* _XLOCALE_STRING2_H */
# 59 "/usr/include/xlocale/_string.h" 3 4
#endif /* _XLOCALE_H_ */
# 60 "/usr/include/xlocale/_string.h" 3 4
# 141 "/usr/include/string.h" 2 3 4
#endif
# 142 "/usr/include/string.h" 3 4
__END_DECLS

#endif /* _STRING_H_ */
# 145 "/usr/include/string.h" 3 4
# 33 "sys/unix/erl_unix_sys.h" 2
#ifndef QNX
#if 0 /* expanded by -frewrite-includes */
#include <memory.h>
#endif /* expanded by -frewrite-includes */
# 34 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/memory.h" 1 3 4
/*
 * Copyright (c) 1988, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)memory.h	8.1 (Berkeley) 6/2/93
 *
 * $FreeBSD: head/include/memory.h 203964 2010-02-16 19:39:50Z imp $
 */

#if 0 /* expanded by -frewrite-includes */
#include <string.h>
#endif /* expanded by -frewrite-includes */
# 34 "/usr/include/memory.h" 3 4
# 35 "/usr/include/memory.h" 3 4
# 35 "sys/unix/erl_unix_sys.h" 2
#endif
# 36 "sys/unix/erl_unix_sys.h"

#if defined(__sun__) && defined(__SVR4) && !defined(__EXTENSIONS__)
#   define __EXTENSIONS__
#if 0 /* expanded by -frewrite-includes */
#   include <sys/types.h>
#endif /* expanded by -frewrite-includes */
# 39 "sys/unix/erl_unix_sys.h"
# 40 "sys/unix/erl_unix_sys.h"
#   undef __EXTENSIONS__
#else
# 42 "sys/unix/erl_unix_sys.h"
#if 0 /* expanded by -frewrite-includes */
#   include <sys/types.h>
#endif /* expanded by -frewrite-includes */
# 42 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/sys/types.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1991, 1993, 1994
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)types.h	8.6 (Berkeley) 2/19/95
 * $FreeBSD: head/sys/sys/types.h 294883 2016-01-27 02:23:54Z jhibbits $
 */

#ifndef _SYS_TYPES_H_
#define	_SYS_TYPES_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/sys/types.h" 3 4
# 42 "/usr/include/sys/types.h" 3 4

/* Machine type dependent parameters. */
#if 0 /* expanded by -frewrite-includes */
#include <machine/endian.h>
#endif /* expanded by -frewrite-includes */
# 44 "/usr/include/sys/types.h" 3 4
# 1 "/usr/include/machine/endian.h" 1 3 4
/*-
 * Copyright (c) 2001 David E. O'Brien
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)endian.h	8.1 (Berkeley) 6/10/93
 * $NetBSD: endian.h,v 1.7 1999/08/21 05:53:51 simonb Exp $
 * $FreeBSD: head/sys/arm/include/endian.h 290648 2015-11-10 12:02:41Z mmel $
 */

#ifndef _ENDIAN_H_
#define	_ENDIAN_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/machine/endian.h" 3 4
# 37 "/usr/include/machine/endian.h" 3 4

/*
 * Definitions for byte order, according to byte significance from low
 * address to high.
 */
#define _LITTLE_ENDIAN  1234    /* LSB first: i386, vax */
#define _BIG_ENDIAN     4321    /* MSB first: 68000, ibm, net */
#define _PDP_ENDIAN     3412    /* LSB first in word, MSW first in long */

#ifdef __ARMEB__
#define _BYTE_ORDER	_BIG_ENDIAN
#else
# 49 "/usr/include/machine/endian.h" 3 4
#define	_BYTE_ORDER	_LITTLE_ENDIAN
#endif /* __ARMEB__ */
# 51 "/usr/include/machine/endian.h" 3 4

#if __BSD_VISIBLE
#define LITTLE_ENDIAN   _LITTLE_ENDIAN
#define BIG_ENDIAN      _BIG_ENDIAN
#define PDP_ENDIAN      _PDP_ENDIAN
#define BYTE_ORDER      _BYTE_ORDER
#endif
# 58 "/usr/include/machine/endian.h" 3 4

#ifdef __ARMEB__
#define _QUAD_HIGHWORD 0
#define _QUAD_LOWWORD 1
#define __ntohl(x)	((__uint32_t)(x))
#define __ntohs(x)	((__uint16_t)(x))
#define __htonl(x)	((__uint32_t)(x))
#define __htons(x)	((__uint16_t)(x))
#else
# 67 "/usr/include/machine/endian.h" 3 4
#define _QUAD_HIGHWORD  1
#define _QUAD_LOWWORD 0
#define __ntohl(x)        (__bswap32(x))
#define __ntohs(x)        (__bswap16(x))
#define __htonl(x)        (__bswap32(x))
#define __htons(x)        (__bswap16(x))
#endif /* __ARMEB__ */
# 74 "/usr/include/machine/endian.h" 3 4

static __inline __uint64_t
__bswap64(__uint64_t _x)
{

	return ((_x >> 56) | ((_x >> 40) & 0xff00) | ((_x >> 24) & 0xff0000) |
	    ((_x >> 8) & 0xff000000) | ((_x << 8) & ((__uint64_t)0xff << 32)) |
	    ((_x << 24) & ((__uint64_t)0xff << 40)) |
	    ((_x << 40) & ((__uint64_t)0xff << 48)) | ((_x << 56)));
}

static __inline __uint32_t
__bswap32_var(__uint32_t v)
{
	__uint32_t t1;

	__asm __volatile("eor %1, %0, %0, ror #16\n"
	    		"bic %1, %1, #0x00ff0000\n"
			"mov %0, %0, ror #8\n"
			"eor %0, %0, %1, lsr #8\n"
			 : "+r" (v), "=r" (t1));

	return (v);
}

static __inline __uint16_t
__bswap16_var(__uint16_t v)
{
	__uint32_t ret = v & 0xffff;

	__asm __volatile(
	    "mov    %0, %0, ror #8\n"
	    "orr    %0, %0, %0, lsr #16\n"
	    "bic    %0, %0, %0, lsl #16"
	    : "+r" (ret));

	return ((__uint16_t)ret);
}

#ifdef __OPTIMIZE__

#define __bswap32_constant(x)	\
    ((((x) & 0xff000000U) >> 24) |	\
     (((x) & 0x00ff0000U) >>  8) |	\
     (((x) & 0x0000ff00U) <<  8) |	\
     (((x) & 0x000000ffU) << 24))

#define __bswap16_constant(x)	\
    ((((x) & 0xff00) >> 8) |		\
     (((x) & 0x00ff) << 8))

#define __bswap16(x)	\
    ((__uint16_t)(__builtin_constant_p(x) ?	\
     __bswap16_constant(x) :			\
     __bswap16_var(x)))

#define __bswap32(x)	\
    ((__uint32_t)(__builtin_constant_p(x) ? 	\
     __bswap32_constant(x) :			\
     __bswap32_var(x)))

#else
# 136 "/usr/include/machine/endian.h" 3 4
#define __bswap16(x)	__bswap16_var(x)
#define __bswap32(x)	__bswap32_var(x)

#endif /* __OPTIMIZE__ */
# 140 "/usr/include/machine/endian.h" 3 4
#endif /* !_ENDIAN_H_ */
# 141 "/usr/include/machine/endian.h" 3 4
# 45 "/usr/include/sys/types.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 45 "/usr/include/sys/types.h" 3 4
# 46 "/usr/include/sys/types.h" 3 4

#if 0 /* expanded by -frewrite-includes */
#include <sys/_pthreadtypes.h>
#endif /* expanded by -frewrite-includes */
# 47 "/usr/include/sys/types.h" 3 4
# 1 "/usr/include/sys/_pthreadtypes.h" 1 3 4
/*
 * Copyright (c) 1993, 1994 by Chris Provenzano, proven@mit.edu
 * Copyright (c) 1995-1998 by John Birrell <jb@cimlogic.com.au>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *  This product includes software developed by Chris Provenzano.
 * 4. The name of Chris Provenzano may not be used to endorse or promote 
 *	  products derived from this software without specific prior written
 *	  permission.
 *
 * THIS SOFTWARE IS PROVIDED BY CHRIS PROVENZANO ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL CHRIS PROVENZANO BE LIABLE FOR ANY 
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/_pthreadtypes.h 189828 2009-03-14 20:10:14Z das $
 */

#ifndef _SYS__PTHREADTYPES_H_
#define _SYS__PTHREADTYPES_H_

/*
 * Forward structure definitions.
 *
 * These are mostly opaque to the user.
 */
struct pthread;
struct pthread_attr;
struct pthread_cond;
struct pthread_cond_attr;
struct pthread_mutex;
struct pthread_mutex_attr;
struct pthread_once;
struct pthread_rwlock;
struct pthread_rwlockattr;
struct pthread_barrier;
struct pthread_barrier_attr;
struct pthread_spinlock;

/*
 * Primitive system data type definitions required by P1003.1c.
 *
 * Note that P1003.1c specifies that there are no defined comparison
 * or assignment operators for the types pthread_attr_t, pthread_cond_t,
 * pthread_condattr_t, pthread_mutex_t, pthread_mutexattr_t.
 */
#ifndef _PTHREAD_T_DECLARED
typedef struct	pthread			*pthread_t;
#define	_PTHREAD_T_DECLARED
#endif
# 68 "/usr/include/sys/_pthreadtypes.h" 3 4
typedef struct	pthread_attr		*pthread_attr_t;
typedef struct	pthread_mutex		*pthread_mutex_t;
typedef struct	pthread_mutex_attr	*pthread_mutexattr_t;
typedef struct	pthread_cond		*pthread_cond_t;
typedef struct	pthread_cond_attr	*pthread_condattr_t;
typedef int     			pthread_key_t;
typedef struct	pthread_once		pthread_once_t;
typedef struct	pthread_rwlock		*pthread_rwlock_t;
typedef struct	pthread_rwlockattr	*pthread_rwlockattr_t;
typedef struct	pthread_barrier		*pthread_barrier_t;
typedef struct	pthread_barrierattr	*pthread_barrierattr_t;
typedef struct	pthread_spinlock	*pthread_spinlock_t;

/*
 * Additional type definitions:
 *
 * Note that P1003.1c reserves the prefixes pthread_ and PTHREAD_ for
 * use in header symbols.
 */
typedef void	*pthread_addr_t;
typedef void	*(*pthread_startroutine_t)(void *);

/*
 * Once definitions.
 */
struct pthread_once {
	int		state;
	pthread_mutex_t	mutex;
};

#endif /* ! _SYS__PTHREADTYPES_H_ */
# 99 "/usr/include/sys/_pthreadtypes.h" 3 4
# 48 "/usr/include/sys/types.h" 2 3 4

#if __BSD_VISIBLE
typedef	unsigned char	u_char;
typedef	unsigned short	u_short;
typedef	unsigned int	u_int;
typedef	unsigned long	u_long;
#ifndef _KERNEL
typedef	unsigned short	ushort;		/* Sys V compatibility */
typedef	unsigned int	uint;		/* Sys V compatibility */
#endif
# 58 "/usr/include/sys/types.h" 3 4
#endif
# 59 "/usr/include/sys/types.h" 3 4

/*
 * XXX POSIX sized integrals that should appear only in <sys/stdint.h>.
 */
#if 0 /* expanded by -frewrite-includes */
#include <sys/_stdint.h>
#endif /* expanded by -frewrite-includes */
# 63 "/usr/include/sys/types.h" 3 4
# 1 "/usr/include/sys/_stdint.h" 1 3 4
/*-
 * Copyright (c) 2011 David E. O'Brien <obrien@FreeBSD.org>
 * Copyright (c) 2001 Mike Barcroft <mike@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/_stdint.h 290492 2015-11-07 12:30:43Z kib $
 */

#ifndef _SYS__STDINT_H_
#define _SYS__STDINT_H_

#ifndef _INT8_T_DECLARED
typedef	__int8_t		int8_t;
#define	_INT8_T_DECLARED
#endif
# 37 "/usr/include/sys/_stdint.h" 3 4

#ifndef _INT16_T_DECLARED
typedef	__int16_t		int16_t;
#define	_INT16_T_DECLARED
#endif
# 42 "/usr/include/sys/_stdint.h" 3 4

#ifndef _INT32_T_DECLARED
typedef	__int32_t		int32_t;
#define	_INT32_T_DECLARED
#endif
# 47 "/usr/include/sys/_stdint.h" 3 4

#ifndef _INT64_T_DECLARED
typedef	__int64_t		int64_t;
#define	_INT64_T_DECLARED
#endif
# 52 "/usr/include/sys/_stdint.h" 3 4

#ifndef _UINT8_T_DECLARED
typedef	__uint8_t		uint8_t;
#define	_UINT8_T_DECLARED
#endif
# 57 "/usr/include/sys/_stdint.h" 3 4

#ifndef _UINT16_T_DECLARED
typedef	__uint16_t		uint16_t;
#define	_UINT16_T_DECLARED
#endif
# 62 "/usr/include/sys/_stdint.h" 3 4

#ifndef _UINT32_T_DECLARED
typedef	__uint32_t		uint32_t;
#define	_UINT32_T_DECLARED
#endif
# 67 "/usr/include/sys/_stdint.h" 3 4

#ifndef _UINT64_T_DECLARED
typedef	__uint64_t		uint64_t;
#define	_UINT64_T_DECLARED
#endif
# 72 "/usr/include/sys/_stdint.h" 3 4

#ifndef _INTPTR_T_DECLARED
typedef	__intptr_t		intptr_t;
#define	_INTPTR_T_DECLARED
#endif
# 77 "/usr/include/sys/_stdint.h" 3 4
#ifndef _UINTPTR_T_DECLARED
typedef	__uintptr_t		uintptr_t;
#define	_UINTPTR_T_DECLARED
#endif
# 81 "/usr/include/sys/_stdint.h" 3 4
#ifndef _INTMAX_T_DECLARED
typedef	__intmax_t		intmax_t;
#define	_INTMAX_T_DECLARED
#endif
# 85 "/usr/include/sys/_stdint.h" 3 4
#ifndef _UINTMAX_T_DECLARED
typedef	__uintmax_t		uintmax_t;
#define	_UINTMAX_T_DECLARED
#endif
# 89 "/usr/include/sys/_stdint.h" 3 4

#endif /* !_SYS__STDINT_H_ */
# 91 "/usr/include/sys/_stdint.h" 3 4
# 64 "/usr/include/sys/types.h" 2 3 4

typedef __uint8_t	u_int8_t;	/* unsigned integrals (deprecated) */
typedef __uint16_t	u_int16_t;
typedef __uint32_t	u_int32_t;
typedef __uint64_t	u_int64_t;

typedef	__uint64_t	u_quad_t;	/* quads (deprecated) */
typedef	__int64_t	quad_t;
typedef	quad_t *	qaddr_t;

typedef	char *		caddr_t;	/* core address */
typedef	const char *	c_caddr_t;	/* core address, pointer to const */

#ifndef _BLKSIZE_T_DECLARED
typedef	__blksize_t	blksize_t;
#define	_BLKSIZE_T_DECLARED
#endif
# 81 "/usr/include/sys/types.h" 3 4

typedef	__cpuwhich_t	cpuwhich_t;
typedef	__cpulevel_t	cpulevel_t;
typedef	__cpusetid_t	cpusetid_t;

#ifndef _BLKCNT_T_DECLARED
typedef	__blkcnt_t	blkcnt_t;
#define	_BLKCNT_T_DECLARED
#endif
# 90 "/usr/include/sys/types.h" 3 4

#ifndef _CLOCK_T_DECLARED
typedef	__clock_t	clock_t;
#define	_CLOCK_T_DECLARED
#endif
# 95 "/usr/include/sys/types.h" 3 4

#ifndef _CLOCKID_T_DECLARED
typedef	__clockid_t	clockid_t;
#define	_CLOCKID_T_DECLARED
#endif
# 100 "/usr/include/sys/types.h" 3 4

typedef	__critical_t	critical_t;	/* Critical section value */
typedef	__int64_t	daddr_t;	/* disk address */

#ifndef _DEV_T_DECLARED
typedef	__dev_t		dev_t;		/* device number or struct cdev */
#define	_DEV_T_DECLARED
#endif
# 108 "/usr/include/sys/types.h" 3 4

#ifndef _FFLAGS_T_DECLARED
typedef	__fflags_t	fflags_t;	/* file flags */
#define	_FFLAGS_T_DECLARED
#endif
# 113 "/usr/include/sys/types.h" 3 4

typedef	__fixpt_t	fixpt_t;	/* fixed point number */

#ifndef _FSBLKCNT_T_DECLARED		/* for statvfs() */
typedef	__fsblkcnt_t	fsblkcnt_t;
typedef	__fsfilcnt_t	fsfilcnt_t;
#define	_FSBLKCNT_T_DECLARED
#endif
# 121 "/usr/include/sys/types.h" 3 4

#ifndef _GID_T_DECLARED
typedef	__gid_t		gid_t;		/* group id */
#define	_GID_T_DECLARED
#endif
# 126 "/usr/include/sys/types.h" 3 4

#ifndef _IN_ADDR_T_DECLARED
typedef	__uint32_t	in_addr_t;	/* base type for internet address */
#define	_IN_ADDR_T_DECLARED
#endif
# 131 "/usr/include/sys/types.h" 3 4

#ifndef _IN_PORT_T_DECLARED
typedef	__uint16_t	in_port_t;
#define	_IN_PORT_T_DECLARED
#endif
# 136 "/usr/include/sys/types.h" 3 4

#ifndef _ID_T_DECLARED
typedef	__id_t		id_t;		/* can hold a uid_t or pid_t */
#define	_ID_T_DECLARED
#endif
# 141 "/usr/include/sys/types.h" 3 4

#ifndef _INO_T_DECLARED
typedef	__ino_t		ino_t;		/* inode number */
#define	_INO_T_DECLARED
#endif
# 146 "/usr/include/sys/types.h" 3 4

#ifndef _KEY_T_DECLARED
typedef	__key_t		key_t;		/* IPC key (for Sys V IPC) */
#define	_KEY_T_DECLARED
#endif
# 151 "/usr/include/sys/types.h" 3 4

#ifndef _LWPID_T_DECLARED
typedef	__lwpid_t	lwpid_t;	/* Thread ID (a.k.a. LWP) */
#define	_LWPID_T_DECLARED
#endif
# 156 "/usr/include/sys/types.h" 3 4

#ifndef _MODE_T_DECLARED
typedef	__mode_t	mode_t;		/* permissions */
#define	_MODE_T_DECLARED
#endif
# 161 "/usr/include/sys/types.h" 3 4

#ifndef _ACCMODE_T_DECLARED
typedef	__accmode_t	accmode_t;	/* access permissions */
#define	_ACCMODE_T_DECLARED
#endif
# 166 "/usr/include/sys/types.h" 3 4

#ifndef _NLINK_T_DECLARED
typedef	__nlink_t	nlink_t;	/* link count */
#define	_NLINK_T_DECLARED
#endif
# 171 "/usr/include/sys/types.h" 3 4

#ifndef _OFF_T_DECLARED
typedef	__off_t		off_t;		/* file offset */
#define	_OFF_T_DECLARED
#endif
# 176 "/usr/include/sys/types.h" 3 4

#ifndef _PID_T_DECLARED
typedef	__pid_t		pid_t;		/* process id */
#define	_PID_T_DECLARED
#endif
# 181 "/usr/include/sys/types.h" 3 4

typedef	__register_t	register_t;

#ifndef _RLIM_T_DECLARED
typedef	__rlim_t	rlim_t;		/* resource limit */
#define	_RLIM_T_DECLARED
#endif
# 188 "/usr/include/sys/types.h" 3 4

typedef	__int64_t	sbintime_t;

typedef	__segsz_t	segsz_t;	/* segment size (in pages) */

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 197 "/usr/include/sys/types.h" 3 4

#ifndef _SSIZE_T_DECLARED
typedef	__ssize_t	ssize_t;
#define	_SSIZE_T_DECLARED
#endif
# 202 "/usr/include/sys/types.h" 3 4

#ifndef _SUSECONDS_T_DECLARED
typedef	__suseconds_t	suseconds_t;	/* microseconds (signed) */
#define	_SUSECONDS_T_DECLARED
#endif
# 207 "/usr/include/sys/types.h" 3 4

#ifndef _TIME_T_DECLARED
typedef	__time_t	time_t;
#define	_TIME_T_DECLARED
#endif
# 212 "/usr/include/sys/types.h" 3 4

#ifndef _TIMER_T_DECLARED
typedef	__timer_t	timer_t;
#define	_TIMER_T_DECLARED
#endif
# 217 "/usr/include/sys/types.h" 3 4

#ifndef _MQD_T_DECLARED
typedef	__mqd_t	mqd_t;
#define	_MQD_T_DECLARED
#endif
# 222 "/usr/include/sys/types.h" 3 4

typedef	__u_register_t	u_register_t;

#ifndef _UID_T_DECLARED
typedef	__uid_t		uid_t;		/* user id */
#define	_UID_T_DECLARED
#endif
# 229 "/usr/include/sys/types.h" 3 4

#ifndef _USECONDS_T_DECLARED
typedef	__useconds_t	useconds_t;	/* microseconds (unsigned) */
#define	_USECONDS_T_DECLARED
#endif
# 234 "/usr/include/sys/types.h" 3 4

#ifndef _CAP_IOCTL_T_DECLARED
#define	_CAP_IOCTL_T_DECLARED
typedef	unsigned long	cap_ioctl_t;
#endif
# 239 "/usr/include/sys/types.h" 3 4

#ifndef _CAP_RIGHTS_T_DECLARED
#define	_CAP_RIGHTS_T_DECLARED
struct cap_rights;

typedef	struct cap_rights	cap_rights_t;
#endif
# 246 "/usr/include/sys/types.h" 3 4

typedef	__vm_offset_t	vm_offset_t;
typedef	__vm_ooffset_t	vm_ooffset_t;
typedef	__vm_paddr_t	vm_paddr_t;
typedef	__vm_pindex_t	vm_pindex_t;
typedef	__vm_size_t	vm_size_t;

typedef __rman_res_t    rman_res_t;

#ifdef _KERNEL
typedef	int		boolean_t;
typedef	struct device	*device_t;
typedef	__intfptr_t	intfptr_t;

/*
 * XXX this is fixed width for historical reasons.  It should have had type
 * __int_fast32_t.  Fixed-width types should not be used unless binary
 * compatibility is essential.  Least-width types should be used even less
 * since they provide smaller benefits.
 *
 * XXX should be MD.
 *
 * XXX this is bogus in -current, but still used for spl*().
 */
typedef	__uint32_t	intrmask_t;	/* Interrupt mask (spl, xxx_imask...) */

typedef	__uintfptr_t	uintfptr_t;
typedef	__uint64_t	uoff_t;
typedef	char		vm_memattr_t;	/* memory attribute codes */
typedef	struct vm_page	*vm_page_t;

#if !defined(__bool_true_false_are_defined) && !defined(__cplusplus)
#define	__bool_true_false_are_defined	1
#define	false	0
#define	true	1
#if __STDC_VERSION__ < 199901L && __GNUC__ < 3 && !defined(__INTEL_COMPILER)
typedef	int	_Bool;
#endif
# 284 "/usr/include/sys/types.h" 3 4
typedef	_Bool	bool;
#endif /* !__bool_true_false_are_defined && !__cplusplus */
# 286 "/usr/include/sys/types.h" 3 4

#define offsetof(type, field) __offsetof(type, field)

#endif /* !_KERNEL */
# 290 "/usr/include/sys/types.h" 3 4

/*
 * The following are all things that really shouldn't exist in this header,
 * since its purpose is to provide typedefs, not miscellaneous doodads.
 */

#ifdef __POPCNT__
#define	__bitcount64(x)	__builtin_popcountll((__uint64_t)(x))
#define	__bitcount32(x)	__builtin_popcount((__uint32_t)(x))
#define	__bitcount16(x)	__builtin_popcount((__uint16_t)(x))
#define	__bitcountl(x)	__builtin_popcountl((unsigned long)(x))
#define	__bitcount(x)	__builtin_popcount((unsigned int)(x))
#else
# 303 "/usr/include/sys/types.h" 3 4
/*
 * Population count algorithm using SWAR approach
 * - "SIMD Within A Register".
 */
static __inline __uint16_t
__bitcount16(__uint16_t _x)
{

	_x = (_x & 0x5555) + ((_x & 0xaaaa) >> 1);
	_x = (_x & 0x3333) + ((_x & 0xcccc) >> 2);
	_x = (_x + (_x >> 4)) & 0x0f0f;
	_x = (_x + (_x >> 8)) & 0x00ff;
	return (_x);
}

static __inline __uint32_t
__bitcount32(__uint32_t _x)
{

	_x = (_x & 0x55555555) + ((_x & 0xaaaaaaaa) >> 1);
	_x = (_x & 0x33333333) + ((_x & 0xcccccccc) >> 2);
	_x = (_x + (_x >> 4)) & 0x0f0f0f0f;
	_x = (_x + (_x >> 8));
	_x = (_x + (_x >> 16)) & 0x000000ff;
	return (_x);
}

#ifdef __LP64__
static __inline __uint64_t
__bitcount64(__uint64_t _x)
{

	_x = (_x & 0x5555555555555555) + ((_x & 0xaaaaaaaaaaaaaaaa) >> 1);
	_x = (_x & 0x3333333333333333) + ((_x & 0xcccccccccccccccc) >> 2);
	_x = (_x + (_x >> 4)) & 0x0f0f0f0f0f0f0f0f;
	_x = (_x + (_x >> 8));
	_x = (_x + (_x >> 16));
	_x = (_x + (_x >> 32)) & 0x000000ff;
	return (_x);
}

#define	__bitcountl(x)	__bitcount64((unsigned long)(x))
#else
# 346 "/usr/include/sys/types.h" 3 4
static __inline __uint64_t
__bitcount64(__uint64_t _x)
{

	return (__bitcount32(_x >> 32) + __bitcount32(_x));
}

#define	__bitcountl(x)	__bitcount32((unsigned long)(x))
#endif
# 355 "/usr/include/sys/types.h" 3 4
#define	__bitcount(x)	__bitcount32((unsigned int)(x))
#endif
# 357 "/usr/include/sys/types.h" 3 4

#if __BSD_VISIBLE

#if 0 /* expanded by -frewrite-includes */
#include <sys/select.h>
#endif /* expanded by -frewrite-includes */
# 360 "/usr/include/sys/types.h" 3 4
# 1 "/usr/include/sys/select.h" 1 3 4
/*-
 * Copyright (c) 1992, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/select.h 265051 2014-04-28 13:42:41Z emaste $
 */

#ifndef _SYS_SELECT_H_
#define	_SYS_SELECT_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 35 "/usr/include/sys/select.h" 3 4
# 36 "/usr/include/sys/select.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/select.h" 3 4
# 37 "/usr/include/sys/select.h" 3 4

#if 0 /* expanded by -frewrite-includes */
#include <sys/_sigset.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/sys/select.h" 3 4
# 1 "/usr/include/sys/_sigset.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1989, 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)signal.h	8.4 (Berkeley) 5/4/95
 * $FreeBSD: head/sys/sys/_sigset.h 139825 2005-01-07 02:29:27Z imp $
 */

#ifndef _SYS__SIGSET_H_
#define	_SYS__SIGSET_H_

/*
 * sigset_t macros.
 */
#define	_SIG_WORDS	4
#define	_SIG_MAXSIG	128
#define	_SIG_IDX(sig)	((sig) - 1)
#define	_SIG_WORD(sig)	(_SIG_IDX(sig) >> 5)
#define	_SIG_BIT(sig)	(1 << (_SIG_IDX(sig) & 31))
#define	_SIG_VALID(sig)	((sig) <= _SIG_MAXSIG && (sig) > 0)

typedef struct __sigset {
	__uint32_t __bits[_SIG_WORDS];
} __sigset_t;

#if defined(_KERNEL) && defined(COMPAT_43)
typedef unsigned int osigset_t;
#endif
# 58 "/usr/include/sys/_sigset.h" 3 4

#endif /* !_SYS__SIGSET_H_ */
# 60 "/usr/include/sys/_sigset.h" 3 4
# 39 "/usr/include/sys/select.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_timeval.h>
#endif /* expanded by -frewrite-includes */
# 39 "/usr/include/sys/select.h" 3 4
# 1 "/usr/include/sys/_timeval.h" 1 3 4
/*-
 * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/_timeval.h 158471 2006-05-12 05:04:46Z jhb $
 */

#ifndef _SYS__TIMEVAL_H_
#define _SYS__TIMEVAL_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 32 "/usr/include/sys/_timeval.h" 3 4
# 33 "/usr/include/sys/_timeval.h" 3 4

#ifndef _SUSECONDS_T_DECLARED
typedef	__suseconds_t	suseconds_t;
#define	_SUSECONDS_T_DECLARED
#endif
# 38 "/usr/include/sys/_timeval.h" 3 4

#ifndef _TIME_T_DECLARED
typedef	__time_t	time_t;
#define	_TIME_T_DECLARED
#endif
# 43 "/usr/include/sys/_timeval.h" 3 4

/*
 * Structure returned by gettimeofday(2) system call, and used in other calls.
 */
struct timeval {
	time_t		tv_sec;		/* seconds */
	suseconds_t	tv_usec;	/* and microseconds */
};

#endif /* !_SYS__TIMEVAL_H_ */
# 53 "/usr/include/sys/_timeval.h" 3 4
# 40 "/usr/include/sys/select.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/timespec.h>
#endif /* expanded by -frewrite-includes */
# 40 "/usr/include/sys/select.h" 3 4
# 1 "/usr/include/sys/timespec.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)time.h	8.5 (Berkeley) 5/4/95
 * from: FreeBSD: src/sys/sys/time.h,v 1.43 2000/03/20 14:09:05 phk Exp
 *	$FreeBSD: head/sys/sys/timespec.h 205792 2010-03-28 13:13:22Z ed $
 */

#ifndef _SYS_TIMESPEC_H_
#define _SYS_TIMESPEC_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/sys/timespec.h" 3 4
# 38 "/usr/include/sys/timespec.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_timespec.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/sys/timespec.h" 3 4
# 1 "/usr/include/sys/_timespec.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)time.h	8.5 (Berkeley) 5/4/95
 * from: FreeBSD: src/sys/sys/time.h,v 1.43 2000/03/20 14:09:05 phk Exp
 *	$FreeBSD: head/sys/sys/_timespec.h 205792 2010-03-28 13:13:22Z ed $
 */

#ifndef _SYS__TIMESPEC_H_
#define	_SYS__TIMESPEC_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/sys/_timespec.h" 3 4
# 38 "/usr/include/sys/_timespec.h" 3 4

#ifndef _TIME_T_DECLARED
typedef	__time_t	time_t;
#define	_TIME_T_DECLARED
#endif
# 43 "/usr/include/sys/_timespec.h" 3 4

struct timespec {
	time_t	tv_sec;		/* seconds */
	long	tv_nsec;	/* and nanoseconds */
};

#endif /* !_SYS__TIMESPEC_H_ */
# 50 "/usr/include/sys/_timespec.h" 3 4
# 39 "/usr/include/sys/timespec.h" 2 3 4

#if __BSD_VISIBLE
#define	TIMEVAL_TO_TIMESPEC(tv, ts)					\
	do {								\
		(ts)->tv_sec = (tv)->tv_sec;				\
		(ts)->tv_nsec = (tv)->tv_usec * 1000;			\
	} while (0)
#define	TIMESPEC_TO_TIMEVAL(tv, ts)					\
	do {								\
		(tv)->tv_sec = (ts)->tv_sec;				\
		(tv)->tv_usec = (ts)->tv_nsec / 1000;			\
	} while (0)

#endif /* __BSD_VISIBLE */
# 53 "/usr/include/sys/timespec.h" 3 4

/*
 * Structure defined by POSIX.1b to be like a itimerval, but with
 * timespecs. Used in the timer_*() system calls.
 */
struct itimerspec {
	struct timespec  it_interval;
	struct timespec  it_value;
};

#endif /* _SYS_TIMESPEC_H_ */
# 64 "/usr/include/sys/timespec.h" 3 4
# 41 "/usr/include/sys/select.h" 2 3 4

typedef	unsigned long	__fd_mask;
#if __BSD_VISIBLE
typedef	__fd_mask	fd_mask;
#endif
# 46 "/usr/include/sys/select.h" 3 4

#ifndef _SIGSET_T_DECLARED
#define	_SIGSET_T_DECLARED
typedef	__sigset_t	sigset_t;
#endif
# 51 "/usr/include/sys/select.h" 3 4

/*
 * Select uses bit masks of file descriptors in longs.  These macros
 * manipulate such bit fields (the filesystem macros use chars).
 * FD_SETSIZE may be defined by the user, but the default here should
 * be enough for most uses.
 */
#ifndef	FD_SETSIZE
#define	FD_SETSIZE	1024
#endif
# 61 "/usr/include/sys/select.h" 3 4

#define	_NFDBITS	(sizeof(__fd_mask) * 8)	/* bits per mask */
#if __BSD_VISIBLE
#define	NFDBITS		_NFDBITS
#endif
# 66 "/usr/include/sys/select.h" 3 4

#ifndef _howmany
#define	_howmany(x, y)	(((x) + ((y) - 1)) / (y))
#endif
# 70 "/usr/include/sys/select.h" 3 4

typedef	struct fd_set {
	__fd_mask	__fds_bits[_howmany(FD_SETSIZE, _NFDBITS)];
} fd_set;
#if __BSD_VISIBLE
#define	fds_bits	__fds_bits
#endif
# 77 "/usr/include/sys/select.h" 3 4

#define	__fdset_mask(n)	((__fd_mask)1 << ((n) % _NFDBITS))
#define	FD_CLR(n, p)	((p)->__fds_bits[(n)/_NFDBITS] &= ~__fdset_mask(n))
#if __BSD_VISIBLE
#define	FD_COPY(f, t)	(void)(*(t) = *(f))
#endif
# 83 "/usr/include/sys/select.h" 3 4
#define	FD_ISSET(n, p)	(((p)->__fds_bits[(n)/_NFDBITS] & __fdset_mask(n)) != 0)
#define	FD_SET(n, p)	((p)->__fds_bits[(n)/_NFDBITS] |= __fdset_mask(n))
#define	FD_ZERO(p) do {					\
	fd_set *_p;					\
	__size_t _n;					\
							\
	_p = (p);					\
	_n = _howmany(FD_SETSIZE, _NFDBITS);		\
	while (_n > 0)					\
		_p->__fds_bits[--_n] = 0;		\
} while (0)

#ifndef _KERNEL

__BEGIN_DECLS
int pselect(int, fd_set *__restrict, fd_set *__restrict, fd_set *__restrict,
	const struct timespec *__restrict, const sigset_t *__restrict);
#ifndef _SELECT_DECLARED
#define	_SELECT_DECLARED
/* XXX missing restrict type-qualifier */
int	select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
#endif
# 105 "/usr/include/sys/select.h" 3 4
__END_DECLS
#endif /* !_KERNEL */
# 107 "/usr/include/sys/select.h" 3 4

#endif /* _SYS_SELECT_H_ */
# 109 "/usr/include/sys/select.h" 3 4
# 361 "/usr/include/sys/types.h" 2 3 4

/*
 * minor() gives a cookie instead of an index since we don't want to
 * change the meanings of bits 0-15 or waste time and space shifting
 * bits 16-31 for devices that don't use them.
 */
#define	major(x)	((int)(((u_int)(x) >> 8)&0xff))	/* major number */
#define	minor(x)	((int)((x)&0xffff00ff))		/* minor number */
#define	makedev(x,y)	((dev_t)(((x) << 8) | (y)))	/* create dev_t */

/*
 * These declarations belong elsewhere, but are repeated here and in
 * <stdio.h> to give broken programs a better chance of working with
 * 64-bit off_t's.
 */
#ifndef _KERNEL
__BEGIN_DECLS
#ifndef _FTRUNCATE_DECLARED
#define	_FTRUNCATE_DECLARED
int	 ftruncate(int, off_t);
#endif
# 382 "/usr/include/sys/types.h" 3 4
#ifndef _LSEEK_DECLARED
#define	_LSEEK_DECLARED
off_t	 lseek(int, off_t, int);
#endif
# 386 "/usr/include/sys/types.h" 3 4
#ifndef _MMAP_DECLARED
#define	_MMAP_DECLARED
void *	 mmap(void *, size_t, int, int, int, off_t);
#endif
# 390 "/usr/include/sys/types.h" 3 4
#ifndef _TRUNCATE_DECLARED
#define	_TRUNCATE_DECLARED
int	 truncate(const char *, off_t);
#endif
# 394 "/usr/include/sys/types.h" 3 4
__END_DECLS
#endif /* !_KERNEL */
# 396 "/usr/include/sys/types.h" 3 4

#endif /* __BSD_VISIBLE */
# 398 "/usr/include/sys/types.h" 3 4

#endif /* !_SYS_TYPES_H_ */
# 400 "/usr/include/sys/types.h" 3 4
# 43 "sys/unix/erl_unix_sys.h" 2
#endif
# 44 "sys/unix/erl_unix_sys.h"
#if 0 /* expanded by -frewrite-includes */
#include <sys/stat.h>
#endif /* expanded by -frewrite-includes */
# 44 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/sys/stat.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)stat.h	8.12 (Berkeley) 6/16/95
 * $FreeBSD: head/sys/sys/stat.h 277610 2015-01-23 21:07:08Z jilles $
 */

#ifndef _SYS_STAT_H_
#define	_SYS_STAT_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/sys/stat.h" 3 4
# 42 "/usr/include/sys/stat.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_timespec.h>
#endif /* expanded by -frewrite-includes */
# 42 "/usr/include/sys/stat.h" 3 4
# 43 "/usr/include/sys/stat.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 43 "/usr/include/sys/stat.h" 3 4
# 44 "/usr/include/sys/stat.h" 3 4

#ifndef _BLKSIZE_T_DECLARED
typedef	__blksize_t	blksize_t;
#define	_BLKSIZE_T_DECLARED
#endif
# 49 "/usr/include/sys/stat.h" 3 4

#ifndef _BLKCNT_T_DECLARED
typedef	__blkcnt_t	blkcnt_t;
#define	_BLKCNT_T_DECLARED
#endif
# 54 "/usr/include/sys/stat.h" 3 4

#ifndef _DEV_T_DECLARED
typedef	__dev_t		dev_t;
#define	_DEV_T_DECLARED
#endif
# 59 "/usr/include/sys/stat.h" 3 4

#ifndef _FFLAGS_T_DECLARED
typedef	__fflags_t	fflags_t;
#define	_FFLAGS_T_DECLARED
#endif
# 64 "/usr/include/sys/stat.h" 3 4

#ifndef _GID_T_DECLARED
typedef	__gid_t		gid_t;
#define	_GID_T_DECLARED
#endif
# 69 "/usr/include/sys/stat.h" 3 4

#ifndef _INO_T_DECLARED
typedef	__ino_t		ino_t;
#define	_INO_T_DECLARED
#endif
# 74 "/usr/include/sys/stat.h" 3 4

#ifndef _MODE_T_DECLARED
typedef	__mode_t	mode_t;
#define	_MODE_T_DECLARED
#endif
# 79 "/usr/include/sys/stat.h" 3 4

#ifndef _NLINK_T_DECLARED
typedef	__nlink_t	nlink_t;
#define	_NLINK_T_DECLARED
#endif
# 84 "/usr/include/sys/stat.h" 3 4

#ifndef _OFF_T_DECLARED
typedef	__off_t		off_t;
#define	_OFF_T_DECLARED
#endif
# 89 "/usr/include/sys/stat.h" 3 4

#ifndef _UID_T_DECLARED
typedef	__uid_t		uid_t;
#define	_UID_T_DECLARED
#endif
# 94 "/usr/include/sys/stat.h" 3 4

#if !defined(_KERNEL) && __BSD_VISIBLE
/*
 * XXX We get miscellaneous namespace pollution with this.
 */
#if 0 /* expanded by -frewrite-includes */
#include <sys/time.h>
#endif /* expanded by -frewrite-includes */
# 99 "/usr/include/sys/stat.h" 3 4
# 1 "/usr/include/sys/time.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)time.h	8.5 (Berkeley) 5/4/95
 * $FreeBSD: head/sys/sys/time.h 275985 2014-12-21 05:07:11Z imp $
 */

#ifndef _SYS_TIME_H_
#define	_SYS_TIME_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_timeval.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/time.h" 3 4
# 37 "/usr/include/sys/time.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/types.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/sys/time.h" 3 4
# 38 "/usr/include/sys/time.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/timespec.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/sys/time.h" 3 4
# 39 "/usr/include/sys/time.h" 3 4

struct timezone {
	int	tz_minuteswest;	/* minutes west of Greenwich */
	int	tz_dsttime;	/* type of dst correction */
};
#define	DST_NONE	0	/* not on dst */
#define	DST_USA		1	/* USA style dst */
#define	DST_AUST	2	/* Australian style dst */
#define	DST_WET		3	/* Western European dst */
#define	DST_MET		4	/* Middle European dst */
#define	DST_EET		5	/* Eastern European dst */
#define	DST_CAN		6	/* Canada */

#if __BSD_VISIBLE
struct bintime {
	time_t	sec;
	uint64_t frac;
};

static __inline void
bintime_addx(struct bintime *_bt, uint64_t _x)
{
	uint64_t _u;

	_u = _bt->frac;
	_bt->frac += _x;
	if (_u > _bt->frac)
		_bt->sec++;
}

static __inline void
bintime_add(struct bintime *_bt, const struct bintime *_bt2)
{
	uint64_t _u;

	_u = _bt->frac;
	_bt->frac += _bt2->frac;
	if (_u > _bt->frac)
		_bt->sec++;
	_bt->sec += _bt2->sec;
}

static __inline void
bintime_sub(struct bintime *_bt, const struct bintime *_bt2)
{
	uint64_t _u;

	_u = _bt->frac;
	_bt->frac -= _bt2->frac;
	if (_u < _bt->frac)
		_bt->sec--;
	_bt->sec -= _bt2->sec;
}

static __inline void
bintime_mul(struct bintime *_bt, u_int _x)
{
	uint64_t _p1, _p2;

	_p1 = (_bt->frac & 0xffffffffull) * _x;
	_p2 = (_bt->frac >> 32) * _x + (_p1 >> 32);
	_bt->sec *= _x;
	_bt->sec += (_p2 >> 32);
	_bt->frac = (_p2 << 32) | (_p1 & 0xffffffffull);
}

static __inline void
bintime_shift(struct bintime *_bt, int _exp)
{

	if (_exp > 0) {
		_bt->sec <<= _exp;
		_bt->sec |= _bt->frac >> (64 - _exp);
		_bt->frac <<= _exp;
	} else if (_exp < 0) {
		_bt->frac >>= -_exp;
		_bt->frac |= (uint64_t)_bt->sec << (64 + _exp);
		_bt->sec >>= -_exp;
	}
}

#define	bintime_clear(a)	((a)->sec = (a)->frac = 0)
#define	bintime_isset(a)	((a)->sec || (a)->frac)
#define	bintime_cmp(a, b, cmp)						\
	(((a)->sec == (b)->sec) ?					\
	    ((a)->frac cmp (b)->frac) :					\
	    ((a)->sec cmp (b)->sec))

#define	SBT_1S	((sbintime_t)1 << 32)
#define	SBT_1M	(SBT_1S * 60)
#define	SBT_1MS	(SBT_1S / 1000)
#define	SBT_1US	(SBT_1S / 1000000)
#define	SBT_1NS	(SBT_1S / 1000000000)
#define	SBT_MAX	0x7fffffffffffffffLL

static __inline int
sbintime_getsec(sbintime_t _sbt)
{

	return (_sbt >> 32);
}

static __inline sbintime_t
bttosbt(const struct bintime _bt)
{

	return (((sbintime_t)_bt.sec << 32) + (_bt.frac >> 32));
}

static __inline struct bintime
sbttobt(sbintime_t _sbt)
{
	struct bintime _bt;

	_bt.sec = _sbt >> 32;
	_bt.frac = _sbt << 32;
	return (_bt);
}

/*-
 * Background information:
 *
 * When converting between timestamps on parallel timescales of differing
 * resolutions it is historical and scientific practice to round down rather
 * than doing 4/5 rounding.
 *
 *   The date changes at midnight, not at noon.
 *
 *   Even at 15:59:59.999999999 it's not four'o'clock.
 *
 *   time_second ticks after N.999999999 not after N.4999999999
 */

static __inline void
bintime2timespec(const struct bintime *_bt, struct timespec *_ts)
{

	_ts->tv_sec = _bt->sec;
	_ts->tv_nsec = ((uint64_t)1000000000 *
	    (uint32_t)(_bt->frac >> 32)) >> 32;
}

static __inline void
timespec2bintime(const struct timespec *_ts, struct bintime *_bt)
{

	_bt->sec = _ts->tv_sec;
	/* 18446744073 = int(2^64 / 1000000000) */
	_bt->frac = _ts->tv_nsec * (uint64_t)18446744073LL;
}

static __inline void
bintime2timeval(const struct bintime *_bt, struct timeval *_tv)
{

	_tv->tv_sec = _bt->sec;
	_tv->tv_usec = ((uint64_t)1000000 * (uint32_t)(_bt->frac >> 32)) >> 32;
}

static __inline void
timeval2bintime(const struct timeval *_tv, struct bintime *_bt)
{

	_bt->sec = _tv->tv_sec;
	/* 18446744073709 = int(2^64 / 1000000) */
	_bt->frac = _tv->tv_usec * (uint64_t)18446744073709LL;
}

static __inline struct timespec
sbttots(sbintime_t _sbt)
{
	struct timespec _ts;

	_ts.tv_sec = _sbt >> 32;
	_ts.tv_nsec = ((uint64_t)1000000000 * (uint32_t)_sbt) >> 32;
	return (_ts);
}

static __inline sbintime_t
tstosbt(struct timespec _ts)
{

	return (((sbintime_t)_ts.tv_sec << 32) +
	    (_ts.tv_nsec * (((uint64_t)1 << 63) / 500000000) >> 32));
}

static __inline struct timeval
sbttotv(sbintime_t _sbt)
{
	struct timeval _tv;

	_tv.tv_sec = _sbt >> 32;
	_tv.tv_usec = ((uint64_t)1000000 * (uint32_t)_sbt) >> 32;
	return (_tv);
}

static __inline sbintime_t
tvtosbt(struct timeval _tv)
{

	return (((sbintime_t)_tv.tv_sec << 32) +
	    (_tv.tv_usec * (((uint64_t)1 << 63) / 500000) >> 32));
}
#endif /* __BSD_VISIBLE */
# 243 "/usr/include/sys/time.h" 3 4

#ifdef _KERNEL

/* Operations on timespecs */
#define	timespecclear(tvp)	((tvp)->tv_sec = (tvp)->tv_nsec = 0)
#define	timespecisset(tvp)	((tvp)->tv_sec || (tvp)->tv_nsec)
#define	timespeccmp(tvp, uvp, cmp)					\
	(((tvp)->tv_sec == (uvp)->tv_sec) ?				\
	    ((tvp)->tv_nsec cmp (uvp)->tv_nsec) :			\
	    ((tvp)->tv_sec cmp (uvp)->tv_sec))
#define	timespecadd(vvp, uvp)						\
	do {								\
		(vvp)->tv_sec += (uvp)->tv_sec;				\
		(vvp)->tv_nsec += (uvp)->tv_nsec;			\
		if ((vvp)->tv_nsec >= 1000000000) {			\
			(vvp)->tv_sec++;				\
			(vvp)->tv_nsec -= 1000000000;			\
		}							\
	} while (0)
#define	timespecsub(vvp, uvp)						\
	do {								\
		(vvp)->tv_sec -= (uvp)->tv_sec;				\
		(vvp)->tv_nsec -= (uvp)->tv_nsec;			\
		if ((vvp)->tv_nsec < 0) {				\
			(vvp)->tv_sec--;				\
			(vvp)->tv_nsec += 1000000000;			\
		}							\
	} while (0)

/* Operations on timevals. */

#define	timevalclear(tvp)		((tvp)->tv_sec = (tvp)->tv_usec = 0)
#define	timevalisset(tvp)		((tvp)->tv_sec || (tvp)->tv_usec)
#define	timevalcmp(tvp, uvp, cmp)					\
	(((tvp)->tv_sec == (uvp)->tv_sec) ?				\
	    ((tvp)->tv_usec cmp (uvp)->tv_usec) :			\
	    ((tvp)->tv_sec cmp (uvp)->tv_sec))

/* timevaladd and timevalsub are not inlined */

#endif /* _KERNEL */
# 284 "/usr/include/sys/time.h" 3 4

#ifndef _KERNEL			/* NetBSD/OpenBSD compatible interfaces */

#define	timerclear(tvp)		((tvp)->tv_sec = (tvp)->tv_usec = 0)
#define	timerisset(tvp)		((tvp)->tv_sec || (tvp)->tv_usec)
#define	timercmp(tvp, uvp, cmp)					\
	(((tvp)->tv_sec == (uvp)->tv_sec) ?				\
	    ((tvp)->tv_usec cmp (uvp)->tv_usec) :			\
	    ((tvp)->tv_sec cmp (uvp)->tv_sec))
#define	timeradd(tvp, uvp, vvp)						\
	do {								\
		(vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec;		\
		(vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec;	\
		if ((vvp)->tv_usec >= 1000000) {			\
			(vvp)->tv_sec++;				\
			(vvp)->tv_usec -= 1000000;			\
		}							\
	} while (0)
#define	timersub(tvp, uvp, vvp)						\
	do {								\
		(vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;		\
		(vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;	\
		if ((vvp)->tv_usec < 0) {				\
			(vvp)->tv_sec--;				\
			(vvp)->tv_usec += 1000000;			\
		}							\
	} while (0)
#endif
# 312 "/usr/include/sys/time.h" 3 4

/*
 * Names of the interval timers, and structure
 * defining a timer setting.
 */
#define	ITIMER_REAL	0
#define	ITIMER_VIRTUAL	1
#define	ITIMER_PROF	2

struct itimerval {
	struct	timeval it_interval;	/* timer interval */
	struct	timeval it_value;	/* current value */
};

/*
 * Getkerninfo clock information structure
 */
struct clockinfo {
	int	hz;		/* clock frequency */
	int	tick;		/* micro-seconds per hz tick */
	int	spare;
	int	stathz;		/* statistics clock frequency */
	int	profhz;		/* profiling clock frequency */
};

/* These macros are also in time.h. */
#ifndef CLOCK_REALTIME
#define	CLOCK_REALTIME	0
#define	CLOCK_VIRTUAL	1
#define	CLOCK_PROF	2
#define	CLOCK_MONOTONIC	4
#define	CLOCK_UPTIME	5		/* FreeBSD-specific. */
#define	CLOCK_UPTIME_PRECISE	7	/* FreeBSD-specific. */
#define	CLOCK_UPTIME_FAST	8	/* FreeBSD-specific. */
#define	CLOCK_REALTIME_PRECISE	9	/* FreeBSD-specific. */
#define	CLOCK_REALTIME_FAST	10	/* FreeBSD-specific. */
#define	CLOCK_MONOTONIC_PRECISE	11	/* FreeBSD-specific. */
#define	CLOCK_MONOTONIC_FAST	12	/* FreeBSD-specific. */
#define	CLOCK_SECOND	13		/* FreeBSD-specific. */
#define	CLOCK_THREAD_CPUTIME_ID	14
#define	CLOCK_PROCESS_CPUTIME_ID	15
#endif
# 354 "/usr/include/sys/time.h" 3 4

#ifndef TIMER_ABSTIME
#define	TIMER_RELTIME	0x0	/* relative timer */
#define	TIMER_ABSTIME	0x1	/* absolute timer */
#endif
# 359 "/usr/include/sys/time.h" 3 4

#if __BSD_VISIBLE
#define	CPUCLOCK_WHICH_PID	0
#define	CPUCLOCK_WHICH_TID	1
#endif
# 364 "/usr/include/sys/time.h" 3 4

#ifdef _KERNEL

/*
 * Kernel to clock driver interface.
 */
void	inittodr(time_t base);
void	resettodr(void);

extern volatile time_t	time_second;
extern volatile time_t	time_uptime;
extern struct bintime boottimebin;
extern struct timeval boottime;
extern struct bintime tc_tick_bt;
extern sbintime_t tc_tick_sbt;
extern struct bintime tick_bt;
extern sbintime_t tick_sbt;
extern int tc_precexp;
extern int tc_timepercentage;
extern struct bintime bt_timethreshold;
extern struct bintime bt_tickthreshold;
extern sbintime_t sbt_timethreshold;
extern sbintime_t sbt_tickthreshold;

/*
 * Functions for looking at our clock: [get]{bin,nano,micro}[up]time()
 *
 * Functions without the "get" prefix returns the best timestamp
 * we can produce in the given format.
 *
 * "bin"   == struct bintime  == seconds + 64 bit fraction of seconds.
 * "nano"  == struct timespec == seconds + nanoseconds.
 * "micro" == struct timeval  == seconds + microseconds.
 *
 * Functions containing "up" returns time relative to boot and
 * should be used for calculating time intervals.
 *
 * Functions without "up" returns UTC time.
 *
 * Functions with the "get" prefix returns a less precise result
 * much faster than the functions without "get" prefix and should
 * be used where a precision of 1/hz seconds is acceptable or where
 * performance is priority. (NB: "precision", _not_ "resolution" !)
 */

void	binuptime(struct bintime *bt);
void	nanouptime(struct timespec *tsp);
void	microuptime(struct timeval *tvp);

static __inline sbintime_t
sbinuptime(void)
{
	struct bintime _bt;

	binuptime(&_bt);
	return (bttosbt(_bt));
}

void	bintime(struct bintime *bt);
void	nanotime(struct timespec *tsp);
void	microtime(struct timeval *tvp);

void	getbinuptime(struct bintime *bt);
void	getnanouptime(struct timespec *tsp);
void	getmicrouptime(struct timeval *tvp);

static __inline sbintime_t
getsbinuptime(void)
{
	struct bintime _bt;

	getbinuptime(&_bt);
	return (bttosbt(_bt));
}

void	getbintime(struct bintime *bt);
void	getnanotime(struct timespec *tsp);
void	getmicrotime(struct timeval *tvp);

/* Other functions */
int	itimerdecr(struct itimerval *itp, int usec);
int	itimerfix(struct timeval *tv);
int	ppsratecheck(struct timeval *, int *, int);
int	ratecheck(struct timeval *, const struct timeval *);
void	timevaladd(struct timeval *t1, const struct timeval *t2);
void	timevalsub(struct timeval *t1, const struct timeval *t2);
int	tvtohz(struct timeval *tv);

#define	TC_DEFAULTPERC		5

#define	BT2FREQ(bt)                                                     \
	(((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) /           \
	    ((bt)->frac >> 1))

#define	SBT2FREQ(sbt)	((SBT_1S + ((sbt) >> 1)) / (sbt))

#define	FREQ2BT(freq, bt)                                               \
{									\
	(bt)->sec = 0;                                                  \
	(bt)->frac = ((uint64_t)0x8000000000000000  / (freq)) << 1;     \
}

#define	TIMESEL(sbt, sbt2)						\
	(((sbt2) >= sbt_timethreshold) ?				\
	    ((*(sbt) = getsbinuptime()), 1) : ((*(sbt) = sbinuptime()), 0))

#else /* !_KERNEL */
# 471 "/usr/include/sys/time.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <time.h>
#endif /* expanded by -frewrite-includes */
# 471 "/usr/include/sys/time.h" 3 4
# 1 "/usr/include/time.h" 1 3 4
/*
 * Copyright (c) 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)time.h	8.3 (Berkeley) 1/21/94
 */

/*
 * $FreeBSD: head/include/time.h 245428 2013-01-14 18:01:19Z dim $
 */

#ifndef _TIME_H_
#define	_TIME_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 44 "/usr/include/time.h" 3 4
# 45 "/usr/include/time.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 45 "/usr/include/time.h" 3 4
# 46 "/usr/include/time.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 46 "/usr/include/time.h" 3 4
# 47 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE > 0 && __POSIX_VISIBLE < 200112 || __BSD_VISIBLE
/*
 * Frequency of the clock ticks reported by times().  Deprecated - use
 * sysconf(_SC_CLK_TCK) instead.  (Removed in 1003.1-2001.)
 */
#define	CLK_TCK		128
#endif
# 55 "/usr/include/time.h" 3 4

/* Frequency of the clock ticks reported by clock().  */
#define	CLOCKS_PER_SEC	128

#ifndef _CLOCK_T_DECLARED
typedef	__clock_t	clock_t;
#define	_CLOCK_T_DECLARED
#endif
# 63 "/usr/include/time.h" 3 4

#ifndef _TIME_T_DECLARED
typedef	__time_t	time_t;
#define	_TIME_T_DECLARED
#endif
# 68 "/usr/include/time.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 73 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE >= 199309
/*
 * New in POSIX 1003.1b-1993.
 */
#ifndef _CLOCKID_T_DECLARED
typedef	__clockid_t	clockid_t;
#define	_CLOCKID_T_DECLARED
#endif
# 82 "/usr/include/time.h" 3 4

#ifndef _TIMER_T_DECLARED
typedef	__timer_t	timer_t;
#define	_TIMER_T_DECLARED
#endif
# 87 "/usr/include/time.h" 3 4

#if 0 /* expanded by -frewrite-includes */
#include <sys/timespec.h>
#endif /* expanded by -frewrite-includes */
# 88 "/usr/include/time.h" 3 4
# 89 "/usr/include/time.h" 3 4
#endif /* __POSIX_VISIBLE >= 199309 */
# 90 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE >= 200112
#ifndef _PID_T_DECLARED
typedef	__pid_t		pid_t;
#define	_PID_T_DECLARED
#endif
# 96 "/usr/include/time.h" 3 4
#endif
# 97 "/usr/include/time.h" 3 4

/* These macros are also in sys/time.h. */
#if !defined(CLOCK_REALTIME) && __POSIX_VISIBLE >= 200112
#define CLOCK_REALTIME	0
#ifdef __BSD_VISIBLE
#define CLOCK_VIRTUAL	1
#define CLOCK_PROF	2
#endif
# 105 "/usr/include/time.h" 3 4
#define CLOCK_MONOTONIC	4
#define CLOCK_UPTIME	5		/* FreeBSD-specific. */
#define CLOCK_UPTIME_PRECISE	7	/* FreeBSD-specific. */
#define CLOCK_UPTIME_FAST	8	/* FreeBSD-specific. */
#define CLOCK_REALTIME_PRECISE	9	/* FreeBSD-specific. */
#define CLOCK_REALTIME_FAST	10	/* FreeBSD-specific. */
#define CLOCK_MONOTONIC_PRECISE	11	/* FreeBSD-specific. */
#define CLOCK_MONOTONIC_FAST	12	/* FreeBSD-specific. */
#define CLOCK_SECOND	13		/* FreeBSD-specific. */
#define CLOCK_THREAD_CPUTIME_ID	14
#define	CLOCK_PROCESS_CPUTIME_ID	15
#endif /* !defined(CLOCK_REALTIME) && __POSIX_VISIBLE >= 200112 */
# 117 "/usr/include/time.h" 3 4

#if !defined(TIMER_ABSTIME) && __POSIX_VISIBLE >= 200112
#if __BSD_VISIBLE
#define TIMER_RELTIME	0x0	/* relative timer */
#endif
# 122 "/usr/include/time.h" 3 4
#define TIMER_ABSTIME	0x1	/* absolute timer */
#endif /* !defined(TIMER_ABSTIME) && __POSIX_VISIBLE >= 200112 */
# 124 "/usr/include/time.h" 3 4

struct tm {
	int	tm_sec;		/* seconds after the minute [0-60] */
	int	tm_min;		/* minutes after the hour [0-59] */
	int	tm_hour;	/* hours since midnight [0-23] */
	int	tm_mday;	/* day of the month [1-31] */
	int	tm_mon;		/* months since January [0-11] */
	int	tm_year;	/* years since 1900 */
	int	tm_wday;	/* days since Sunday [0-6] */
	int	tm_yday;	/* days since January 1 [0-365] */
	int	tm_isdst;	/* Daylight Savings Time flag */
	long	tm_gmtoff;	/* offset from UTC in seconds */
	char	*tm_zone;	/* timezone abbreviation */
};

#if __POSIX_VISIBLE
extern char *tzname[];
#endif
# 142 "/usr/include/time.h" 3 4

__BEGIN_DECLS
char *asctime(const struct tm *);
clock_t clock(void);
char *ctime(const time_t *);
double difftime(time_t, time_t);
/* XXX missing: getdate() */
struct tm *gmtime(const time_t *);
struct tm *localtime(const time_t *);
time_t mktime(struct tm *);
size_t strftime(char * __restrict, size_t, const char * __restrict,
    const struct tm * __restrict);
time_t time(time_t *);
#if __POSIX_VISIBLE >= 200112
struct sigevent;
int timer_create(clockid_t, struct sigevent *__restrict, timer_t *__restrict);
int timer_delete(timer_t);
int timer_gettime(timer_t, struct itimerspec *);
int timer_getoverrun(timer_t);
int timer_settime(timer_t, int, const struct itimerspec *__restrict,
	struct itimerspec *__restrict);
#endif
# 164 "/usr/include/time.h" 3 4
#if __POSIX_VISIBLE
void tzset(void);
#endif
# 167 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE >= 199309
int clock_getres(clockid_t, struct timespec *);
int clock_gettime(clockid_t, struct timespec *);
int clock_settime(clockid_t, const struct timespec *);
/* XXX missing: clock_nanosleep() */
int nanosleep(const struct timespec *, struct timespec *);
#endif /* __POSIX_VISIBLE >= 199309 */
# 175 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE >= 200112
int clock_getcpuclockid(pid_t, clockid_t *);
#endif
# 179 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE >= 199506
char *asctime_r(const struct tm *, char *);
char *ctime_r(const time_t *, char *);
struct tm *gmtime_r(const time_t *, struct tm *);
struct tm *localtime_r(const time_t *, struct tm *);
#endif
# 186 "/usr/include/time.h" 3 4

#if __XSI_VISIBLE
char *strptime(const char * __restrict, const char * __restrict,
    struct tm * __restrict);
#endif
# 191 "/usr/include/time.h" 3 4

#if __BSD_VISIBLE
char *timezone(int, int);	/* XXX XSI conflict */
void tzsetwall(void);
time_t timelocal(struct tm * const);
time_t timegm(struct tm * const);
#endif /* __BSD_VISIBLE */
# 198 "/usr/include/time.h" 3 4

#if __POSIX_VISIBLE >= 200809 || defined(_XLOCALE_H_)
#if 0 /* expanded by -frewrite-includes */
#include <xlocale/_time.h>
#endif /* expanded by -frewrite-includes */
# 200 "/usr/include/time.h" 3 4
# 1 "/usr/include/xlocale/_time.h" 1 3 4
/*-
 * Copyright (c) 2011, 2012 The FreeBSD Foundation
 * All rights reserved.
 *
 * This software was developed by David Chisnall under sponsorship from
 * the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/include/xlocale/_time.h 231673 2012-02-14 12:03:23Z theraven $
 */

#ifndef _LOCALE_T_DEFINED
#define _LOCALE_T_DEFINED
typedef struct	_xlocale *locale_t;
#endif
# 36 "/usr/include/xlocale/_time.h" 3 4

/*
 * This file is included from both locale.h and xlocale.h.  We need to expose
 * the declarations unconditionally if we are included from xlocale.h, but only
 * if we are in POSIX2008 mode if included from locale.h.
 */
#ifndef _XLOCALE_LOCALE1_H
#define _XLOCALE_LOCALE1_H

size_t	 strftime_l(char * __restrict, size_t, const char * __restrict,
	    const struct tm * __restrict, locale_t) __strftimelike(3, 0);

#endif /* _XLOCALE_LOCALE1_H */
# 49 "/usr/include/xlocale/_time.h" 3 4

#ifdef _XLOCALE_H_
#ifndef _XLOCALE_LOCALE2_H
#define _XLOCALE_LOCALE2_H

char	*strptime_l(const char * __restrict, const char * __restrict,
           struct tm * __restrict, locale_t);

#endif /* _XLOCALE_LOCALE2_H */
# 58 "/usr/include/xlocale/_time.h" 3 4
#endif /* _XLOCALE_H_ */
# 59 "/usr/include/xlocale/_time.h" 3 4
# 201 "/usr/include/time.h" 2 3 4
#endif
# 202 "/usr/include/time.h" 3 4
__END_DECLS

#endif /* !_TIME_H_ */
# 205 "/usr/include/time.h" 3 4
# 472 "/usr/include/sys/time.h" 2 3 4

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 473 "/usr/include/sys/time.h" 3 4
# 474 "/usr/include/sys/time.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/select.h>
#endif /* expanded by -frewrite-includes */
# 474 "/usr/include/sys/time.h" 3 4
# 475 "/usr/include/sys/time.h" 3 4

__BEGIN_DECLS
int	setitimer(int, const struct itimerval *, struct itimerval *);
int	utimes(const char *, const struct timeval *);

#if __BSD_VISIBLE
int	adjtime(const struct timeval *, struct timeval *);
int	clock_getcpuclockid2(id_t, int, clockid_t *);
int	futimes(int, const struct timeval *);
int	futimesat(int, const char *, const struct timeval [2]);
int	lutimes(const char *, const struct timeval *);
int	settimeofday(const struct timeval *, const struct timezone *);
#endif
# 488 "/usr/include/sys/time.h" 3 4

#if __XSI_VISIBLE
int	getitimer(int, struct itimerval *);
int	gettimeofday(struct timeval *, struct timezone *);
#endif
# 493 "/usr/include/sys/time.h" 3 4

__END_DECLS

#endif /* !_KERNEL */
# 497 "/usr/include/sys/time.h" 3 4

#endif /* !_SYS_TIME_H_ */
# 499 "/usr/include/sys/time.h" 3 4
# 100 "/usr/include/sys/stat.h" 2 3 4
#endif
# 101 "/usr/include/sys/stat.h" 3 4

#ifdef _KERNEL
struct ostat {
	__uint16_t st_dev;		/* inode's device */
	ino_t	  st_ino;		/* inode's number */
	mode_t	  st_mode;		/* inode protection mode */
	nlink_t	  st_nlink;		/* number of hard links */
	__uint16_t st_uid;		/* user ID of the file's owner */
	__uint16_t st_gid;		/* group ID of the file's group */
	__uint16_t st_rdev;		/* device type */
	__int32_t st_size;		/* file size, in bytes */
	struct	timespec st_atim;	/* time of last access */
	struct	timespec st_mtim;	/* time of last data modification */
	struct	timespec st_ctim;	/* time of last file status change */
	__int32_t st_blksize;		/* optimal blocksize for I/O */
	__int32_t st_blocks;		/* blocks allocated for file */
	fflags_t  st_flags;		/* user defined flags for file */
	__uint32_t st_gen;		/* file generation number */
};
#endif
# 121 "/usr/include/sys/stat.h" 3 4

struct stat {
	__dev_t   st_dev;		/* inode's device */
	ino_t	  st_ino;		/* inode's number */
	mode_t	  st_mode;		/* inode protection mode */
	nlink_t	  st_nlink;		/* number of hard links */
	uid_t	  st_uid;		/* user ID of the file's owner */
	gid_t	  st_gid;		/* group ID of the file's group */
	__dev_t   st_rdev;		/* device type */
	struct	timespec st_atim;	/* time of last access */
	struct	timespec st_mtim;	/* time of last data modification */
	struct	timespec st_ctim;	/* time of last file status change */
	off_t	  st_size;		/* file size, in bytes */
	blkcnt_t st_blocks;		/* blocks allocated for file */
	blksize_t st_blksize;		/* optimal blocksize for I/O */
	fflags_t  st_flags;		/* user defined flags for file */
	__uint32_t st_gen;		/* file generation number */
	__int32_t st_lspare;
	struct timespec st_birthtim;	/* time of file creation */
	/*
	 * Explicitly pad st_birthtim to 16 bytes so that the size of
	 * struct stat is backwards compatible.  We use bitfields instead
	 * of an array of chars so that this doesn't require a C99 compiler
	 * to compile if the size of the padding is 0.  We use 2 bitfields
	 * to cover up to 64 bits on 32-bit machines.  We assume that
	 * CHAR_BIT is 8...
	 */
	unsigned int :(8 / 2) * (16 - (int)sizeof(struct timespec));
	unsigned int :(8 / 2) * (16 - (int)sizeof(struct timespec));
};

#ifdef _KERNEL
struct nstat {
	__dev_t   st_dev;		/* inode's device */
	ino_t	  st_ino;		/* inode's number */
	__uint32_t st_mode;		/* inode protection mode */
	__uint32_t st_nlink;		/* number of hard links */
	uid_t	  st_uid;		/* user ID of the file's owner */
	gid_t	  st_gid;		/* group ID of the file's group */
	__dev_t   st_rdev;		/* device type */
	struct	timespec st_atim;	/* time of last access */
	struct	timespec st_mtim;	/* time of last data modification */
	struct	timespec st_ctim;	/* time of last file status change */
	off_t	  st_size;		/* file size, in bytes */
	blkcnt_t st_blocks;		/* blocks allocated for file */
	blksize_t st_blksize;		/* optimal blocksize for I/O */
	fflags_t  st_flags;		/* user defined flags for file */
	__uint32_t st_gen;		/* file generation number */
	struct timespec st_birthtim;	/* time of file creation */
	/*
	 * See above about the following padding.
	 */
	unsigned int :(8 / 2) * (16 - (int)sizeof(struct timespec));
	unsigned int :(8 / 2) * (16 - (int)sizeof(struct timespec));
};
#endif
# 177 "/usr/include/sys/stat.h" 3 4

#ifndef _KERNEL
#define	st_atime		st_atim.tv_sec
#define	st_mtime		st_mtim.tv_sec
#define	st_ctime		st_ctim.tv_sec
#if __BSD_VISIBLE
#define	st_birthtime		st_birthtim.tv_sec
#endif
# 185 "/usr/include/sys/stat.h" 3 4

/* For compatibility. */
#if __BSD_VISIBLE
#define	st_atimespec		st_atim
#define	st_mtimespec		st_mtim
#define	st_ctimespec		st_ctim
#define	st_birthtimespec	st_birthtim
#endif
# 193 "/usr/include/sys/stat.h" 3 4
#endif /* !_KERNEL */
# 194 "/usr/include/sys/stat.h" 3 4

#define	S_ISUID	0004000			/* set user id on execution */
#define	S_ISGID	0002000			/* set group id on execution */
#if __BSD_VISIBLE
#define	S_ISTXT	0001000			/* sticky bit */
#endif
# 200 "/usr/include/sys/stat.h" 3 4

#define	S_IRWXU	0000700			/* RWX mask for owner */
#define	S_IRUSR	0000400			/* R for owner */
#define	S_IWUSR	0000200			/* W for owner */
#define	S_IXUSR	0000100			/* X for owner */

#if __BSD_VISIBLE
#define	S_IREAD		S_IRUSR
#define	S_IWRITE	S_IWUSR
#define	S_IEXEC		S_IXUSR
#endif
# 211 "/usr/include/sys/stat.h" 3 4

#define	S_IRWXG	0000070			/* RWX mask for group */
#define	S_IRGRP	0000040			/* R for group */
#define	S_IWGRP	0000020			/* W for group */
#define	S_IXGRP	0000010			/* X for group */

#define	S_IRWXO	0000007			/* RWX mask for other */
#define	S_IROTH	0000004			/* R for other */
#define	S_IWOTH	0000002			/* W for other */
#define	S_IXOTH	0000001			/* X for other */

#if __XSI_VISIBLE
#define	S_IFMT	 0170000		/* type of file mask */
#define	S_IFIFO	 0010000		/* named pipe (fifo) */
#define	S_IFCHR	 0020000		/* character special */
#define	S_IFDIR	 0040000		/* directory */
#define	S_IFBLK	 0060000		/* block special */
#define	S_IFREG	 0100000		/* regular */
#define	S_IFLNK	 0120000		/* symbolic link */
#define	S_IFSOCK 0140000		/* socket */
#define	S_ISVTX	 0001000		/* save swapped text even after use */
#endif
# 233 "/usr/include/sys/stat.h" 3 4
#if __BSD_VISIBLE
#define	S_IFWHT  0160000		/* whiteout */
#endif
# 236 "/usr/include/sys/stat.h" 3 4

#define	S_ISDIR(m)	(((m) & 0170000) == 0040000)	/* directory */
#define	S_ISCHR(m)	(((m) & 0170000) == 0020000)	/* char special */
#define	S_ISBLK(m)	(((m) & 0170000) == 0060000)	/* block special */
#define	S_ISREG(m)	(((m) & 0170000) == 0100000)	/* regular file */
#define	S_ISFIFO(m)	(((m) & 0170000) == 0010000)	/* fifo or socket */
#if __POSIX_VISIBLE >= 200112
#define	S_ISLNK(m)	(((m) & 0170000) == 0120000)	/* symbolic link */
#define	S_ISSOCK(m)	(((m) & 0170000) == 0140000)	/* socket */
#endif
# 246 "/usr/include/sys/stat.h" 3 4
#if __BSD_VISIBLE
#define	S_ISWHT(m)	(((m) & 0170000) == 0160000)	/* whiteout */
#endif
# 249 "/usr/include/sys/stat.h" 3 4

#if __BSD_VISIBLE
#define	ACCESSPERMS	(S_IRWXU|S_IRWXG|S_IRWXO)	/* 0777 */
							/* 7777 */
#define	ALLPERMS	(S_ISUID|S_ISGID|S_ISTXT|S_IRWXU|S_IRWXG|S_IRWXO)
							/* 0666 */
#define	DEFFILEMODE	(S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH)

#define S_BLKSIZE	512		/* block size used in the stat struct */

/*
 * Definitions of flags stored in file flags word.
 *
 * Super-user and owner changeable flags.
 */
#define	UF_SETTABLE	0x0000ffff	/* mask of owner changeable flags */
#define	UF_NODUMP	0x00000001	/* do not dump file */
#define	UF_IMMUTABLE	0x00000002	/* file may not be changed */
#define	UF_APPEND	0x00000004	/* writes to file may only append */
#define	UF_OPAQUE	0x00000008	/* directory is opaque wrt. union */
#define	UF_NOUNLINK	0x00000010	/* file may not be removed or renamed */
/*
 * These two bits are defined in MacOS X.  They are not currently used in
 * FreeBSD.
 */
#if 0
#define	UF_COMPRESSED	0x00000020	/* file is compressed */
#define	UF_TRACKED	0x00000040	/* renames and deletes are tracked */
#endif
# 278 "/usr/include/sys/stat.h" 3 4

#define	UF_SYSTEM	0x00000080	/* Windows system file bit */
#define	UF_SPARSE	0x00000100	/* sparse file */
#define	UF_OFFLINE	0x00000200	/* file is offline */
#define	UF_REPARSE	0x00000400	/* Windows reparse point file bit */
#define	UF_ARCHIVE	0x00000800	/* file needs to be archived */
#define	UF_READONLY	0x00001000	/* Windows readonly file bit */
/* This is the same as the MacOS X definition of UF_HIDDEN. */
#define	UF_HIDDEN	0x00008000	/* file is hidden */

/*
 * Super-user changeable flags.
 */
#define	SF_SETTABLE	0xffff0000	/* mask of superuser changeable flags */
#define	SF_ARCHIVED	0x00010000	/* file is archived */
#define	SF_IMMUTABLE	0x00020000	/* file may not be changed */
#define	SF_APPEND	0x00040000	/* writes to file may only append */
#define	SF_NOUNLINK	0x00100000	/* file may not be removed or renamed */
#define	SF_SNAPSHOT	0x00200000	/* snapshot inode */

#ifdef _KERNEL
/*
 * Shorthand abbreviations of above.
 */
#define	OPAQUE		(UF_OPAQUE)
#define	APPEND		(UF_APPEND | SF_APPEND)
#define	IMMUTABLE	(UF_IMMUTABLE | SF_IMMUTABLE)
#define	NOUNLINK	(UF_NOUNLINK | SF_NOUNLINK)
#endif
# 307 "/usr/include/sys/stat.h" 3 4

#endif /* __BSD_VISIBLE */
# 309 "/usr/include/sys/stat.h" 3 4

#if __POSIX_VISIBLE >= 200809
#define	UTIME_NOW	-1
#define	UTIME_OMIT	-2
#endif
# 314 "/usr/include/sys/stat.h" 3 4

#ifndef _KERNEL
__BEGIN_DECLS
#if __BSD_VISIBLE
int	chflags(const char *, unsigned long);
int	chflagsat(int, const char *, unsigned long, int);
#endif
# 321 "/usr/include/sys/stat.h" 3 4
int	chmod(const char *, mode_t);
#if __BSD_VISIBLE
int	fchflags(int, unsigned long);
#endif
# 325 "/usr/include/sys/stat.h" 3 4
#if __POSIX_VISIBLE >= 200112
int	fchmod(int, mode_t);
#endif
# 328 "/usr/include/sys/stat.h" 3 4
#if __POSIX_VISIBLE >= 200809
int	fchmodat(int, const char *, mode_t, int);
int	futimens(int fd, const struct timespec times[2]);
int	utimensat(int fd, const char *path, const struct timespec times[2],
		int flag);
#endif
# 334 "/usr/include/sys/stat.h" 3 4
int	fstat(int, struct stat *);
#if __BSD_VISIBLE
int	lchflags(const char *, unsigned long);
int	lchmod(const char *, mode_t);
#endif
# 339 "/usr/include/sys/stat.h" 3 4
#if __POSIX_VISIBLE >= 200112
int	lstat(const char * __restrict, struct stat * __restrict);
#endif
# 342 "/usr/include/sys/stat.h" 3 4
int	mkdir(const char *, mode_t);
int	mkfifo(const char *, mode_t);
#if !defined(_MKNOD_DECLARED) && __XSI_VISIBLE
int	mknod(const char *, mode_t, dev_t);
#define	_MKNOD_DECLARED
#endif
# 348 "/usr/include/sys/stat.h" 3 4
int	stat(const char * __restrict, struct stat * __restrict);
mode_t	umask(mode_t);
#if __POSIX_VISIBLE >= 200809
int	fstatat(int, const char *, struct stat *, int);
int	mkdirat(int, const char *, mode_t);
int	mkfifoat(int, const char *, mode_t);
#endif
# 355 "/usr/include/sys/stat.h" 3 4
#if __XSI_VISIBLE >= 700
int	mknodat(int, const char *, mode_t, dev_t);
#endif
# 358 "/usr/include/sys/stat.h" 3 4
__END_DECLS
#endif /* !_KERNEL */
# 360 "/usr/include/sys/stat.h" 3 4

#endif /* !_SYS_STAT_H_ */
# 362 "/usr/include/sys/stat.h" 3 4
# 45 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <sys/param.h>
#endif /* expanded by -frewrite-includes */
# 45 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/sys/param.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)param.h	8.3 (Berkeley) 4/4/95
 * $FreeBSD: head/sys/sys/param.h 298486 2016-04-22 18:16:12Z hselasky $
 */

#ifndef _SYS_PARAM_H_
#define _SYS_PARAM_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/sys/param.h" 3 4
# 42 "/usr/include/sys/param.h" 3 4

#define	BSD	199506		/* System version (year & month). */
#define BSD4_3	1
#define BSD4_4	1

/* 
 * __FreeBSD_version numbers are documented in the Porter's Handbook.
 * If you bump the version for any reason, you should update the documentation
 * there.
 * Currently this lives here in the doc/ repository:
 *
 *	head/en_US.ISO8859-1/books/porters-handbook/versions/chapter.xml
 *
 * scheme is:  <major><two digit minor>Rxx
 *		'R' is in the range 0 to 4 if this is a release branch or
 *		x.0-CURRENT before RELENG_*_0 is created, otherwise 'R' is
 *		in the range 5 to 9.
 */
#undef __FreeBSD_version
#define __FreeBSD_version 1100106	/* Master, propagated to newvers */

/*
 * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
 * which by definition is always true on FreeBSD. This macro is also defined
 * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD.
 *
 * It is tempting to use this macro in userland code when we want to enable
 * kernel-specific routines, and in fact it's fine to do this in code that
 * is part of FreeBSD itself.  However, be aware that as presence of this
 * macro is still not widespread (e.g. older FreeBSD versions, 3rd party
 * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in
 * external applications without also checking for __FreeBSD__ as an
 * alternative.
 */
#undef __FreeBSD_kernel__
#define __FreeBSD_kernel__

#ifdef _KERNEL
#define	P_OSREL_SIGWAIT			700000
#define	P_OSREL_SIGSEGV			700004
#define	P_OSREL_MAP_ANON		800104
#define	P_OSREL_MAP_FSTRICT		1100036
#define	P_OSREL_SHUTDOWN_ENOTCONN	1100077

#define	P_OSREL_MAJOR(x)		((x) / 100000)
#endif
# 88 "/usr/include/sys/param.h" 3 4

#ifndef LOCORE
#if 0 /* expanded by -frewrite-includes */
#include <sys/types.h>
#endif /* expanded by -frewrite-includes */
# 90 "/usr/include/sys/param.h" 3 4
# 91 "/usr/include/sys/param.h" 3 4
#endif
# 92 "/usr/include/sys/param.h" 3 4

/*
 * Machine-independent constants (some used in following include files).
 * Redefined constants are from POSIX 1003.1 limits file.
 *
 * MAXCOMLEN should be >= sizeof(ac_comm) (see <acct.h>)
 */
#if 0 /* expanded by -frewrite-includes */
#include <sys/syslimits.h>
#endif /* expanded by -frewrite-includes */
# 99 "/usr/include/sys/param.h" 3 4
# 100 "/usr/include/sys/param.h" 3 4

#define	MAXCOMLEN	19		/* max command name remembered */
#define	MAXINTERP	PATH_MAX	/* max interpreter file name length */
#define	MAXLOGNAME	33		/* max login name length (incl. NUL) */
#define	MAXUPRC		CHILD_MAX	/* max simultaneous processes */
#define	NCARGS		ARG_MAX		/* max bytes for an exec function */
#define	NGROUPS		(NGROUPS_MAX+1)	/* max number groups */
#define	NOFILE		OPEN_MAX	/* max open files per process */
#define	NOGROUP		65535		/* marker for empty group set member */
#define MAXHOSTNAMELEN	256		/* max hostname size */
#define SPECNAMELEN	63		/* max length of devicename */

/* More types and definitions used throughout the kernel. */
#ifdef _KERNEL
#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 114 "/usr/include/sys/param.h" 3 4
# 115 "/usr/include/sys/param.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/errno.h>
#endif /* expanded by -frewrite-includes */
# 115 "/usr/include/sys/param.h" 3 4
# 116 "/usr/include/sys/param.h" 3 4
#ifndef LOCORE
#if 0 /* expanded by -frewrite-includes */
#include <sys/time.h>
#endif /* expanded by -frewrite-includes */
# 117 "/usr/include/sys/param.h" 3 4
# 118 "/usr/include/sys/param.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/priority.h>
#endif /* expanded by -frewrite-includes */
# 118 "/usr/include/sys/param.h" 3 4
# 119 "/usr/include/sys/param.h" 3 4
#endif
# 120 "/usr/include/sys/param.h" 3 4

#ifndef FALSE
#define	FALSE	0
#endif
# 124 "/usr/include/sys/param.h" 3 4
#ifndef TRUE
#define	TRUE	1
#endif
# 127 "/usr/include/sys/param.h" 3 4
#endif
# 128 "/usr/include/sys/param.h" 3 4

#ifndef _KERNEL
/* Signals. */
#if 0 /* expanded by -frewrite-includes */
#include <sys/signal.h>
#endif /* expanded by -frewrite-includes */
# 131 "/usr/include/sys/param.h" 3 4
# 1 "/usr/include/sys/signal.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1989, 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)signal.h	8.4 (Berkeley) 5/4/95
 * $FreeBSD: head/sys/sys/signal.h 295561 2016-02-12 07:38:19Z kib $
 */

#ifndef _SYS_SIGNAL_H_
#define	_SYS_SIGNAL_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/sys/signal.h" 3 4
# 42 "/usr/include/sys/signal.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 42 "/usr/include/sys/signal.h" 3 4
# 43 "/usr/include/sys/signal.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_sigset.h>
#endif /* expanded by -frewrite-includes */
# 43 "/usr/include/sys/signal.h" 3 4
# 44 "/usr/include/sys/signal.h" 3 4

#if 0 /* expanded by -frewrite-includes */
#include <machine/_limits.h>	/* __MINSIGSTKSZ */
#endif /* expanded by -frewrite-includes */
# 45 "/usr/include/sys/signal.h" 3 4
# 46 "/usr/include/sys/signal.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/signal.h>	/* sig_atomic_t; trap codes; sigcontext */
#endif /* expanded by -frewrite-includes */
# 46 "/usr/include/sys/signal.h" 3 4
# 1 "/usr/include/machine/signal.h" 1 3 4
/*-
 * Copyright (c) 1986, 1989, 1991, 1993
 *      The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)signal.h    8.1 (Berkeley) 6/11/93
 *	from: FreeBSD: src/sys/i386/include/signal.h,v 1.13 2000/11/09
 *	from: FreeBSD: src/sys/sparc64/include/signal.h,v 1.6 2001/09/30 18:52:17
 * $FreeBSD: head/sys/arm/include/signal.h 248153 2013-03-11 10:56:46Z cognet $
 */

#ifndef	_MACHINE_SIGNAL_H_
#define	_MACHINE_SIGNAL_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/machine/signal.h" 3 4
# 39 "/usr/include/machine/signal.h" 3 4

typedef	long sig_atomic_t;

#if __BSD_VISIBLE

struct sigcontext {
	int _dummy;
};

#endif
# 49 "/usr/include/machine/signal.h" 3 4

#endif /* !_MACHINE_SIGNAL_H_ */
# 51 "/usr/include/machine/signal.h" 3 4
# 47 "/usr/include/sys/signal.h" 2 3 4

/*
 * System defined signals.
 */
#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SIGHUP		1	/* hangup */
#endif
# 54 "/usr/include/sys/signal.h" 3 4
#define	SIGINT		2	/* interrupt */
#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SIGQUIT		3	/* quit */
#endif
# 58 "/usr/include/sys/signal.h" 3 4
#define	SIGILL		4	/* illegal instr. (not reset when caught) */
#if __XSI_VISIBLE
#define	SIGTRAP		5	/* trace trap (not reset when caught) */
#endif
# 62 "/usr/include/sys/signal.h" 3 4
#define	SIGABRT		6	/* abort() */
#if __BSD_VISIBLE
#define	SIGIOT		SIGABRT	/* compatibility */
#define	SIGEMT		7	/* EMT instruction */
#endif
# 67 "/usr/include/sys/signal.h" 3 4
#define	SIGFPE		8	/* floating point exception */
#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SIGKILL		9	/* kill (cannot be caught or ignored) */
#endif
# 71 "/usr/include/sys/signal.h" 3 4
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#define	SIGBUS		10	/* bus error */
#endif
# 74 "/usr/include/sys/signal.h" 3 4
#define	SIGSEGV		11	/* segmentation violation */
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#define	SIGSYS		12	/* non-existent system call invoked */
#endif
# 78 "/usr/include/sys/signal.h" 3 4
#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SIGPIPE		13	/* write on a pipe with no one to read it */
#define	SIGALRM		14	/* alarm clock */
#endif
# 82 "/usr/include/sys/signal.h" 3 4
#define	SIGTERM		15	/* software termination signal from kill */
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#define	SIGURG		16	/* urgent condition on IO channel */
#endif
# 86 "/usr/include/sys/signal.h" 3 4
#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SIGSTOP		17	/* sendable stop signal not from tty */
#define	SIGTSTP		18	/* stop signal from tty */
#define	SIGCONT		19	/* continue a stopped process */
#define	SIGCHLD		20	/* to parent on child stop or exit */
#define	SIGTTIN		21	/* to readers pgrp upon background tty read */
#define	SIGTTOU		22	/* like TTIN if (tp->t_local&LTOSTOP) */
#endif
# 94 "/usr/include/sys/signal.h" 3 4
#if __BSD_VISIBLE
#define	SIGIO		23	/* input/output possible signal */
#endif
# 97 "/usr/include/sys/signal.h" 3 4
#if __XSI_VISIBLE
#define	SIGXCPU		24	/* exceeded CPU time limit */
#define	SIGXFSZ		25	/* exceeded file size limit */
#define	SIGVTALRM	26	/* virtual time alarm */
#define	SIGPROF		27	/* profiling time alarm */
#endif
# 103 "/usr/include/sys/signal.h" 3 4
#if __BSD_VISIBLE
#define	SIGWINCH	28	/* window size changes */
#define	SIGINFO		29	/* information request */
#endif
# 107 "/usr/include/sys/signal.h" 3 4
#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SIGUSR1		30	/* user defined signal 1 */
#define	SIGUSR2		31	/* user defined signal 2 */
#endif
# 111 "/usr/include/sys/signal.h" 3 4
#if __BSD_VISIBLE
#define	SIGTHR		32	/* reserved by thread library. */
#define	SIGLWP		SIGTHR
#define	SIGLIBRT	33	/* reserved by real-time library. */
#endif
# 116 "/usr/include/sys/signal.h" 3 4

#define	SIGRTMIN	65
#define	SIGRTMAX	126

#define	SIG_DFL		((__sighandler_t *)0)
#define	SIG_IGN		((__sighandler_t *)1)
#define	SIG_ERR		((__sighandler_t *)-1)
/* #define	SIG_CATCH	((__sighandler_t *)2) See signalvar.h */
#define SIG_HOLD        ((__sighandler_t *)3)

/*
 * Type of a signal handling function.
 *
 * Language spec sez signal handlers take exactly one arg, even though we
 * actually supply three.  Ugh!
 *
 * We don't try to hide the difference by leaving out the args because
 * that would cause warnings about conformant programs.  Nonconformant
 * programs can avoid the warnings by casting to (__sighandler_t *) or
 * sig_t before calling signal() or assigning to sa_handler or sv_handler.
 *
 * The kernel should reverse the cast before calling the function.  It
 * has no way to do this, but on most machines 1-arg and 3-arg functions
 * have the same calling protocol so there is no problem in practice.
 * A bit in sa_flags could be used to specify the number of args.
 */
typedef	void __sighandler_t(int);

#if __POSIX_VISIBLE || __XSI_VISIBLE
#ifndef _SIGSET_T_DECLARED
#define	_SIGSET_T_DECLARED
typedef	__sigset_t	sigset_t;
#endif
# 149 "/usr/include/sys/signal.h" 3 4
#endif
# 150 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE >= 199309 || __XSI_VISIBLE >= 500
union sigval {
	/* Members as suggested by Annex C of POSIX 1003.1b. */
	int	sival_int;
	void	*sival_ptr;
	/* 6.0 compatibility */
	int     sigval_int;
	void    *sigval_ptr;
};
#endif
# 161 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE >= 199309
struct sigevent {
	int	sigev_notify;		/* Notification type */
	int	sigev_signo;		/* Signal number */
	union sigval sigev_value;	/* Signal value */
	union {
		__lwpid_t	_threadid;
		struct {
			void (*_function)(union sigval);
			void *_attribute; /* pthread_attr_t * */
		} _sigev_thread;
		unsigned short _kevent_flags;
		long __spare__[8];
	} _sigev_un;
};

#if __BSD_VISIBLE
#define	sigev_notify_kqueue		sigev_signo
#define	sigev_notify_kevent_flags	_sigev_un._kevent_flags
#define	sigev_notify_thread_id		_sigev_un._threadid
#endif
# 183 "/usr/include/sys/signal.h" 3 4
#define	sigev_notify_function		_sigev_un._sigev_thread._function
#define	sigev_notify_attributes		_sigev_un._sigev_thread._attribute

#define	SIGEV_NONE	0		/* No async notification. */
#define	SIGEV_SIGNAL	1		/* Generate a queued signal. */
#define	SIGEV_THREAD	2		/* Call back from another pthread. */
#if __BSD_VISIBLE
#define	SIGEV_KEVENT	3		/* Generate a kevent. */
#define	SIGEV_THREAD_ID	4		/* Send signal to a kernel thread. */
#endif
# 193 "/usr/include/sys/signal.h" 3 4
#endif /* __POSIX_VISIBLE >= 199309 */
# 194 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE >= 199309 || __XSI_VISIBLE
typedef	struct __siginfo {
	int	si_signo;		/* signal number */
	int	si_errno;		/* errno association */
	/*
	 * Cause of signal, one of the SI_ macros or signal-specific
	 * values, i.e. one of the FPE_... values for SIGFPE.  This
	 * value is equivalent to the second argument to an old-style
	 * FreeBSD signal handler.
	 */
	int	si_code;		/* signal code */
	__pid_t	si_pid;			/* sending process */
	__uid_t	si_uid;			/* sender's ruid */
	int	si_status;		/* exit value */
	void	*si_addr;		/* faulting instruction */
	union sigval si_value;		/* signal value */
	union	{
		struct {
			int	_trapno;/* machine specific trap code */
		} _fault;
		struct {
			int	_timerid;
			int	_overrun;
		} _timer;
		struct {
			int	_mqd;
		} _mesgq;
		struct {
			long	_band;		/* band event for SIGPOLL */
		} _poll;			/* was this ever used ? */
		struct {
			long	__spare1__;
			int	__spare2__[7];
		} __spare__;
	} _reason;
} siginfo_t;

#define si_trapno	_reason._fault._trapno
#define si_timerid	_reason._timer._timerid
#define si_overrun	_reason._timer._overrun
#define si_mqd		_reason._mesgq._mqd
#define si_band		_reason._poll._band

/** si_code **/
/* codes for SIGILL */
#define ILL_ILLOPC 	1	/* Illegal opcode.			*/
#define ILL_ILLOPN 	2	/* Illegal operand.			*/
#define ILL_ILLADR 	3	/* Illegal addressing mode.		*/
#define ILL_ILLTRP 	4	/* Illegal trap.			*/
#define ILL_PRVOPC 	5	/* Privileged opcode.			*/
#define ILL_PRVREG 	6	/* Privileged register.			*/
#define ILL_COPROC 	7	/* Coprocessor error.			*/
#define ILL_BADSTK 	8	/* Internal stack error.		*/

/* codes for SIGBUS */
#define BUS_ADRALN	1	/* Invalid address alignment.		*/
#define BUS_ADRERR	2	/* Nonexistent physical address.	*/
#define BUS_OBJERR	3	/* Object-specific hardware error.	*/

/* codes for SIGSEGV */
#define SEGV_MAPERR	1	/* Address not mapped to object.	*/
#define SEGV_ACCERR	2	/* Invalid permissions for mapped	*/
				/* object.				*/

/* codes for SIGFPE */
#define FPE_INTOVF	1	/* Integer overflow.			*/
#define FPE_INTDIV	2	/* Integer divide by zero.		*/
#define FPE_FLTDIV	3	/* Floating point divide by zero.	*/
#define FPE_FLTOVF	4	/* Floating point overflow.		*/
#define FPE_FLTUND	5	/* Floating point underflow.		*/
#define FPE_FLTRES	6	/* Floating point inexact result.	*/
#define FPE_FLTINV	7	/* Invalid floating point operation.	*/
#define FPE_FLTSUB	8	/* Subscript out of range.		*/

/* codes for SIGTRAP */
#define TRAP_BRKPT	1	/* Process breakpoint.			*/
#define TRAP_TRACE	2	/* Process trace trap.			*/
#define	TRAP_DTRACE	3	/* DTrace induced trap.			*/

/* codes for SIGCHLD */
#define CLD_EXITED	1	/* Child has exited			*/
#define CLD_KILLED	2	/* Child has terminated abnormally but	*/
				/* did not create a core file		*/
#define CLD_DUMPED	3	/* Child has terminated abnormally and	*/
				/* created a core file			*/
#define CLD_TRAPPED	4	/* Traced child has trapped		*/
#define CLD_STOPPED	5	/* Child has stopped			*/
#define CLD_CONTINUED	6	/* Stopped child has continued		*/

/* codes for SIGPOLL */
#define POLL_IN		1	/* Data input available			*/
#define POLL_OUT	2	/* Output buffers available		*/
#define POLL_MSG	3	/* Input message available		*/
#define POLL_ERR	4	/* I/O Error				*/
#define POLL_PRI	5	/* High priority input available	*/
#define POLL_HUP	6	/* Device disconnected			*/

#endif
# 293 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE || __XSI_VISIBLE
struct __siginfo;

/*
 * Signal vector "template" used in sigaction call.
 */
struct sigaction {
	union {
		void    (*__sa_handler)(int);
		void    (*__sa_sigaction)(int, struct __siginfo *, void *);
	} __sigaction_u;		/* signal handler */
	int	sa_flags;		/* see signal options below */
	sigset_t sa_mask;		/* signal mask to apply */
};

#define	sa_handler	__sigaction_u.__sa_handler
#endif
# 311 "/usr/include/sys/signal.h" 3 4

#if __XSI_VISIBLE
/* If SA_SIGINFO is set, sa_sigaction must be used instead of sa_handler. */
#define	sa_sigaction	__sigaction_u.__sa_sigaction
#endif
# 316 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SA_NOCLDSTOP	0x0008	/* do not generate SIGCHLD on child stop */
#endif /* __POSIX_VISIBLE || __XSI_VISIBLE */
# 320 "/usr/include/sys/signal.h" 3 4

#if __XSI_VISIBLE
#define	SA_ONSTACK	0x0001	/* take signal on signal stack */
#define	SA_RESTART	0x0002	/* restart system call on signal return */
#define	SA_RESETHAND	0x0004	/* reset to SIG_DFL when taking signal */
#define	SA_NODEFER	0x0010	/* don't mask the signal we're delivering */
#define	SA_NOCLDWAIT	0x0020	/* don't keep zombies around */
#define	SA_SIGINFO	0x0040	/* signal handler with SA_SIGINFO args */
#endif
# 329 "/usr/include/sys/signal.h" 3 4

#if __BSD_VISIBLE
#define	NSIG		32	/* number of old signals (counting 0) */
#endif
# 333 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE || __XSI_VISIBLE
#define	SI_NOINFO	0		/* No signal info besides si_signo. */
#define	SI_USER		0x10001		/* Signal sent by kill(). */
#define	SI_QUEUE	0x10002		/* Signal sent by the sigqueue(). */
#define	SI_TIMER	0x10003		/* Signal generated by expiration of */
					/* a timer set by timer_settime(). */
#define	SI_ASYNCIO	0x10004		/* Signal generated by completion of */
					/* an asynchronous I/O request.*/
#define	SI_MESGQ	0x10005		/* Signal generated by arrival of a */
					/* message on an empty message queue. */
#define	SI_KERNEL	0x10006
#define	SI_LWP		0x10007		/* Signal sent by thr_kill */
#endif
# 347 "/usr/include/sys/signal.h" 3 4
#if __BSD_VISIBLE
#define	SI_UNDEFINED	0
#endif
# 350 "/usr/include/sys/signal.h" 3 4

#if __BSD_VISIBLE
typedef	__sighandler_t	*sig_t;	/* type of pointer to a signal function */
typedef	void __siginfohandler_t(int, struct __siginfo *, void *);
#endif
# 355 "/usr/include/sys/signal.h" 3 4

#if __XSI_VISIBLE
#if __BSD_VISIBLE
#define	__stack_t sigaltstack
#endif
# 360 "/usr/include/sys/signal.h" 3 4
typedef	struct __stack_t stack_t;

#define	SS_ONSTACK	0x0001	/* take signal on alternate stack */
#define	SS_DISABLE	0x0004	/* disable taking signals on alternate stack */
#define	MINSIGSTKSZ	__MINSIGSTKSZ		/* minimum stack size */
#define	SIGSTKSZ	(MINSIGSTKSZ + 32768)	/* recommended stack size */
#endif
# 367 "/usr/include/sys/signal.h" 3 4

/*
 * Structure used in sigaltstack call.  Its definition is always
 * needed for __ucontext.  If __BSD_VISIBLE is defined, the structure
 * tag is actually sigaltstack.
 */
struct __stack_t {
	void	*ss_sp;			/* signal stack base */
	__size_t ss_size;		/* signal stack length */
	int	ss_flags;		/* SS_DISABLE and/or SS_ONSTACK */
};

#if __BSD_VISIBLE
/*
 * 4.3 compatibility:
 * Signal vector "template" used in sigvec call.
 */
struct sigvec {
	__sighandler_t *sv_handler;	/* signal handler */
	int	sv_mask;		/* signal mask to apply */
	int	sv_flags;		/* see signal options below */
};

#define	SV_ONSTACK	SA_ONSTACK
#define	SV_INTERRUPT	SA_RESTART	/* same bit, opposite sense */
#define	SV_RESETHAND	SA_RESETHAND
#define	SV_NODEFER	SA_NODEFER
#define	SV_NOCLDSTOP	SA_NOCLDSTOP
#define	SV_SIGINFO	SA_SIGINFO
#define	sv_onstack	sv_flags	/* isn't compatibility wonderful! */
#endif
# 398 "/usr/include/sys/signal.h" 3 4

/* Keep this in one place only */
#if defined(_KERNEL) && defined(COMPAT_43) && \
    !defined(__i386__)
struct osigcontext {
	int _not_used;
};
#endif
# 406 "/usr/include/sys/signal.h" 3 4

#if __XSI_VISIBLE
/*
 * Structure used in sigstack call.
 */
struct sigstack {
	void	*ss_sp;			/* signal stack pointer */
	int	ss_onstack;		/* current status */
};
#endif
# 416 "/usr/include/sys/signal.h" 3 4

#if __BSD_VISIBLE || __POSIX_VISIBLE > 0 && __POSIX_VISIBLE <= 200112
/*
 * Macro for converting signal number to a mask suitable for
 * sigblock().
 */
#define	sigmask(m)	(1 << ((m)-1))
#endif
# 424 "/usr/include/sys/signal.h" 3 4

#if __BSD_VISIBLE
#define	BADSIG		SIG_ERR
#endif
# 428 "/usr/include/sys/signal.h" 3 4

#if __POSIX_VISIBLE || __XSI_VISIBLE
/*
 * Flags for sigprocmask:
 */
#define	SIG_BLOCK	1	/* block specified signal set */
#define	SIG_UNBLOCK	2	/* unblock specified signal set */
#define	SIG_SETMASK	3	/* set specified signal set */
#endif
# 437 "/usr/include/sys/signal.h" 3 4

/*
 * For historical reasons; programs expect signal's return value to be
 * defined by <sys/signal.h>.
 */
__BEGIN_DECLS
__sighandler_t *signal(int, __sighandler_t *);
__END_DECLS

#endif /* !_SYS_SIGNAL_H_ */
# 447 "/usr/include/sys/signal.h" 3 4
# 132 "/usr/include/sys/param.h" 2 3 4
#endif
# 133 "/usr/include/sys/param.h" 3 4

/* Machine type dependent parameters. */
#if 0 /* expanded by -frewrite-includes */
#include <machine/param.h>
#endif /* expanded by -frewrite-includes */
# 135 "/usr/include/sys/param.h" 3 4
# 1 "/usr/include/machine/param.h" 1 3 4
/*-
 * Copyright (c) 2001 David E. O'Brien
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from: @(#)param.h	5.8 (Berkeley) 6/28/91
 * $FreeBSD: head/sys/arm/include/param.h 295043 2016-01-29 16:42:03Z skra $
 */

#ifndef _ARM_INCLUDE_PARAM_H_
#define	_ARM_INCLUDE_PARAM_H_

/*
 * Machine dependent constants for StrongARM
 */

#if 0 /* expanded by -frewrite-includes */
#include <machine/_align.h>
#endif /* expanded by -frewrite-includes */
# 48 "/usr/include/machine/param.h" 3 4
# 1 "/usr/include/machine/_align.h" 1 3 4
/*-
 * Copyright (c) 2001 David E. O'Brien
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from: @(#)param.h	5.8 (Berkeley) 6/28/91
 * $FreeBSD: head/sys/arm/include/_align.h 196994 2009-09-08 20:45:40Z phk $
 */

#ifndef _ARM_INCLUDE__ALIGN_H_
#define	_ARM_INCLUDE__ALIGN_H_

/*
 * Round p (pointer or byte index) up to a correctly-aligned value
 * for all data types (int, long, ...).   The result is unsigned int
 * and must be cast to any desired pointer type.
 */
#define	_ALIGNBYTES	(sizeof(int) - 1)
#define	_ALIGN(p)	(((unsigned)(p) + _ALIGNBYTES) & ~_ALIGNBYTES)

#endif /* !_ARM_INCLUDE__ALIGN_H_ */
# 53 "/usr/include/machine/_align.h" 3 4
# 49 "/usr/include/machine/param.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/acle-compat.h>
#endif /* expanded by -frewrite-includes */
# 49 "/usr/include/machine/param.h" 3 4
# 1 "/usr/include/machine/acle-compat.h" 1 3 4
/*
 * Copyright (c) 2014 ARM Ltd
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the company may not be used to endorse or promote
 *    products derived from this software without specific prior written
 *    permission.
 *
 * THIS SOFTWARE IS PROVIDED BY ARM LTD ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL ARM LTD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/arm/include/acle-compat.h 269956 2014-08-14 04:20:13Z imp $
 */

#ifndef __ARM_ARCH

/* ACLE standardises a set of pre-defines that describe the ARM architecture.
   These were mostly implemented in GCC around GCC-4.8; older versions
   have no, or only partial support.  To provide a level of backwards
   compatibility we try to work out what the definitions should be, given
   the older pre-defines that GCC did produce.  This isn't complete, but
   it should be enough for use by routines that depend on this header.  */

/* No need to handle ARMv8, GCC had ACLE support before that.  */

#define __ARM_ACLE 101

# ifdef __ARM_ARCH_7__
/* The common subset of ARMv7 in all profiles.  */
#  define __ARM_ARCH 7
#  define __ARM_ARCH_ISA_THUMB 2
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_LDREX 7
#  define __ARM_FEATURE_UNALIGNED
# endif
# 52 "/usr/include/machine/acle-compat.h" 3 4

# if defined (__ARM_ARCH_7A__) || defined (__ARM_ARCH_7R__)
#  define __ARM_ARCH 7
#  define __ARM_ARCH_ISA_THUMB 2
#  define __ARM_ARCH_ISA_ARM
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_SIMD32
#  define __ARM_FEATURE_DSP
#  define __ARM_FEATURE_QBIT
#  define __ARM_FEATURE_SAT
#  define __ARM_FEATURE_LDREX 15
#  define __ARM_FEATURE_UNALIGNED
#  ifdef __ARM_ARCH_7A__
#   define __ARM_ARCH_PROFILE 'A'
#  else
# 67 "/usr/include/machine/acle-compat.h" 3 4
#   define __ARM_ARCH_PROFILE 'R'
#  endif
# 69 "/usr/include/machine/acle-compat.h" 3 4
# endif
# 70 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_7EM__
#  define __ARM_ARCH 7
#  define __ARM_ARCH_ISA_THUMB 2
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_SIMD32
#  define __ARM_FEATURE_DSP
#  define __ARM_FEATURE_QBIT
#  define __ARM_FEATURE_SAT
#  define __ARM_FEATURE_LDREX 7
#  define __ARM_FEATURE_UNALIGNED
#  define __ARM_ARCH_PROFILE 'M'
# endif
# 83 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_7M__
#  define __ARM_ARCH 7
#  define __ARM_ARCH_ISA_THUMB 2
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_QBIT
#  define __ARM_FEATURE_SAT
#  define __ARM_FEATURE_LDREX 7
#  define __ARM_FEATURE_UNALIGNED
#  define __ARM_ARCH_PROFILE 'M'
# endif
# 94 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_6T2__
#  define __ARM_ARCH 6
#  define __ARM_ARCH_ISA_THUMB 2
#  define __ARM_ARCH_ISA_ARM
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_SIMD32
#  define __ARM_FEATURE_DSP
#  define __ARM_FEATURE_QBIT
#  define __ARM_FEATURE_SAT
#  define __ARM_FEATURE_LDREX 4
#  define __ARM_FEATURE_UNALIGNED
# endif
# 107 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_6M__
#  define __ARM_ARCH 6
#  define __ARM_ARCH_ISA_THUMB 1
#  define __ARM_ARCH_PROFILE 'M'
# endif
# 113 "/usr/include/machine/acle-compat.h" 3 4

# if defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) \
  || defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6Z__) \
  || defined (__ARM_ARCH_6ZK__)
#  define __ARM_ARCH 6
#  define __ARM_ARCH_ISA_THUMB 1
#  define __ARM_ARCH_ISA_ARM
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_SIMD32
#  define __ARM_FEATURE_DSP
#  define __ARM_FEATURE_QBIT
#  define __ARM_FEATURE_SAT
#  define __ARM_FEATURE_UNALIGNED
#  ifndef __thumb__
#   if defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6ZK__)
#    define __ARM_FEATURE_LDREX 15
#   else
# 130 "/usr/include/machine/acle-compat.h" 3 4
#    define __ARM_FEATURE_LDREX 4
#   endif
# 132 "/usr/include/machine/acle-compat.h" 3 4
#  endif
# 133 "/usr/include/machine/acle-compat.h" 3 4
# endif
# 134 "/usr/include/machine/acle-compat.h" 3 4

# if defined (__ARM_ARCH_5TE__) || defined (__ARM_ARCH_5E__)
#  define __ARM_ARCH 5
#  define __ARM_ARCH_ISA_ARM
#  ifdef __ARM_ARCH_5TE__
#   define __ARM_ARCH_ISA_THUMB 1
#  endif
# 141 "/usr/include/machine/acle-compat.h" 3 4
#  define __ARM_FEATURE_CLZ
#  define __ARM_FEATURE_DSP
# endif
# 144 "/usr/include/machine/acle-compat.h" 3 4

# if defined (__ARM_ARCH_5T__) || defined (__ARM_ARCH_5__)
#  define __ARM_ARCH 5
#  define __ARM_ARCH_ISA_ARM
#  ifdef __ARM_ARCH_5TE__
#   define __ARM_ARCH_ISA_THUMB 1
#  endif
# 151 "/usr/include/machine/acle-compat.h" 3 4
#  define __ARM_FEATURE_CLZ
# endif
# 153 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_4T__
#  define __ARM_ARCH 4
#  define __ARM_ARCH_ISA_ARM
#  define __ARM_ARCH_ISA_THUMB 1
# endif
# 159 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_4__
#  define __ARM_ARCH 4
#  define __ARM_ARCH_ISA_ARM
# endif
# 164 "/usr/include/machine/acle-compat.h" 3 4

# if defined (__ARM_ARCH_3__) || defined (__ARM_ARCH_3M__)
#  define __ARM_ARCH 3
#  define __ARM_ARCH_ISA_ARM
# endif
# 169 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARM_ARCH_2__
#  define __ARM_ARCH 2
#  define __ARM_ARCH_ISA_ARM
# endif
# 174 "/usr/include/machine/acle-compat.h" 3 4

# ifdef __ARMEB__
#  define __ARM_BIG_ENDIAN
# endif
# 178 "/usr/include/machine/acle-compat.h" 3 4

/* If we still don't know what the target architecture is, then we're
   probably not using GCC.  */
# ifndef __ARM_ARCH
#  error Unable to determine architecture version.
# endif
# 184 "/usr/include/machine/acle-compat.h" 3 4

#endif /* __ARM_ARCH  */
# 186 "/usr/include/machine/acle-compat.h" 3 4
# 50 "/usr/include/machine/param.h" 2 3 4

#define STACKALIGNBYTES	(8 - 1)
#define STACKALIGN(p)	((u_int)(p) & ~STACKALIGNBYTES)

#define __PCI_REROUTE_INTERRUPT

#if __ARM_ARCH >= 6
#define	_V6_SUFFIX "v6"
#else
# 59 "/usr/include/machine/param.h" 3 4
#define	_V6_SUFFIX ""
#endif
# 61 "/usr/include/machine/param.h" 3 4

#ifdef __ARM_PCS_VFP
#define	_HF_SUFFIX "hf"
#else
# 65 "/usr/include/machine/param.h" 3 4
#define	_HF_SUFFIX ""
#endif
# 67 "/usr/include/machine/param.h" 3 4

#ifdef __ARM_BIG_ENDIAN
#define	_EB_SUFFIX "eb"
#else
# 71 "/usr/include/machine/param.h" 3 4
#define	_EB_SUFFIX ""
#endif
# 73 "/usr/include/machine/param.h" 3 4

#ifndef MACHINE
#define	MACHINE		"arm"
#endif
# 77 "/usr/include/machine/param.h" 3 4
#ifndef MACHINE_ARCH
#define	MACHINE_ARCH	"arm" _V6_SUFFIX _HF_SUFFIX _EB_SUFFIX
#endif
# 80 "/usr/include/machine/param.h" 3 4

#if defined(SMP) || defined(KLD_MODULE)
#ifndef MAXCPU
#define	MAXCPU		4
#endif
# 85 "/usr/include/machine/param.h" 3 4
#else
# 86 "/usr/include/machine/param.h" 3 4
#define	MAXCPU		1
#endif /* SMP || KLD_MODULE */
# 88 "/usr/include/machine/param.h" 3 4

#ifndef MAXMEMDOM
#define	MAXMEMDOM	1
#endif
# 92 "/usr/include/machine/param.h" 3 4

#define	ALIGNBYTES	_ALIGNBYTES
#define	ALIGN(p)	_ALIGN(p)
/*
 * ALIGNED_POINTER is a boolean macro that checks whether an address
 * is valid to fetch data elements of type t from on this architecture.
 * This does not reflect the optimal alignment, just the possibility
 * (within reasonable limits).
 */
#define	ALIGNED_POINTER(p, t)	((((unsigned)(p)) & (sizeof(t)-1)) == 0)

/*
 * CACHE_LINE_SIZE is the compile-time maximum cache line size for an
 * architecture.  It should be used with appropriate caution.
 */
#define	CACHE_LINE_SHIFT	6
#define	CACHE_LINE_SIZE		(1 << CACHE_LINE_SHIFT)

#define	PAGE_SHIFT	12
#define	PAGE_SIZE	(1 << PAGE_SHIFT)	/* Page size */
#define	PAGE_MASK	(PAGE_SIZE - 1)

#define PDR_SHIFT	20 /* log2(NBPDR) */
#define NBPDR		(1 << PDR_SHIFT)
#define PDRMASK		(NBPDR - 1)
#define NPDEPG          (1 << (32 - PDR_SHIFT))

#define	MAXPAGESIZES	2		/* maximum number of supported page sizes */

#ifndef KSTACK_PAGES
#define KSTACK_PAGES    2
#endif /* !KSTACK_PAGES */
# 124 "/usr/include/machine/param.h" 3 4

#ifndef FPCONTEXTSIZE
#define FPCONTEXTSIZE	(0x100)
#endif
# 128 "/usr/include/machine/param.h" 3 4

#ifndef KSTACK_GUARD_PAGES
#define KSTACK_GUARD_PAGES	1
#endif /* !KSTACK_GUARD_PAGES */
# 132 "/usr/include/machine/param.h" 3 4

#define USPACE_SVC_STACK_TOP		(kstack_pages * PAGE_SIZE)

/*
 * Mach derived conversion macros
 */
#define	trunc_page(x)		((x) & ~PAGE_MASK)
#define	round_page(x)		(((x) + PAGE_MASK) & ~PAGE_MASK)
#define	trunc_1mpage(x)		((unsigned)(x) & ~PDRMASK)
#define	round_1mpage(x)		((((unsigned)(x)) + PDRMASK) & ~PDRMASK)

#define	atop(x)			((unsigned)(x) >> PAGE_SHIFT)
#define	ptoa(x)			((unsigned)(x) << PAGE_SHIFT)

#define	arm32_btop(x)		((unsigned)(x) >> PAGE_SHIFT)
#define	arm32_ptob(x)		((unsigned)(x) << PAGE_SHIFT)

#define	pgtok(x)		((x) * (PAGE_SIZE / 1024))

#endif /* !_ARM_INCLUDE_PARAM_H_ */
# 152 "/usr/include/machine/param.h" 3 4
# 136 "/usr/include/sys/param.h" 2 3 4
#ifndef _KERNEL
#if 0 /* expanded by -frewrite-includes */
#include <sys/limits.h>
#endif /* expanded by -frewrite-includes */
# 137 "/usr/include/sys/param.h" 3 4
# 138 "/usr/include/sys/param.h" 3 4
#endif
# 139 "/usr/include/sys/param.h" 3 4

#ifndef DEV_BSHIFT
#define	DEV_BSHIFT	9		/* log2(DEV_BSIZE) */
#endif
# 143 "/usr/include/sys/param.h" 3 4
#define	DEV_BSIZE	(1<<DEV_BSHIFT)

#ifndef BLKDEV_IOSIZE
#define BLKDEV_IOSIZE  PAGE_SIZE	/* default block device I/O size */
#endif
# 148 "/usr/include/sys/param.h" 3 4
#ifndef DFLTPHYS
#define DFLTPHYS	(64 * 1024)	/* default max raw I/O transfer size */
#endif
# 151 "/usr/include/sys/param.h" 3 4
#ifndef MAXPHYS
#define MAXPHYS		(128 * 1024)	/* max raw I/O transfer size */
#endif
# 154 "/usr/include/sys/param.h" 3 4
#ifndef MAXDUMPPGS
#define MAXDUMPPGS	(DFLTPHYS/PAGE_SIZE)
#endif
# 157 "/usr/include/sys/param.h" 3 4

/*
 * Constants related to network buffer management.
 * MCLBYTES must be no larger than PAGE_SIZE.
 */
#ifndef	MSIZE
#define	MSIZE		256		/* size of an mbuf */
#endif
# 165 "/usr/include/sys/param.h" 3 4

#ifndef	MCLSHIFT
#define MCLSHIFT	11		/* convert bytes to mbuf clusters */
#endif	/* MCLSHIFT */
# 169 "/usr/include/sys/param.h" 3 4

#define MCLBYTES	(1 << MCLSHIFT)	/* size of an mbuf cluster */

#if PAGE_SIZE < 2048
#define	MJUMPAGESIZE	MCLBYTES
#elif PAGE_SIZE <= 8192
# 175 "/usr/include/sys/param.h" 3 4
#define	MJUMPAGESIZE	PAGE_SIZE
#else
# 177 "/usr/include/sys/param.h" 3 4
#define	MJUMPAGESIZE	(8 * 1024)
#endif
# 179 "/usr/include/sys/param.h" 3 4

#define	MJUM9BYTES	(9 * 1024)	/* jumbo cluster 9k */
#define	MJUM16BYTES	(16 * 1024)	/* jumbo cluster 16k */

/*
 * Some macros for units conversion
 */

/* clicks to bytes */
#ifndef ctob
#define ctob(x)	((x)<<PAGE_SHIFT)
#endif
# 191 "/usr/include/sys/param.h" 3 4

/* bytes to clicks */
#ifndef btoc
#define btoc(x)	(((vm_offset_t)(x)+PAGE_MASK)>>PAGE_SHIFT)
#endif
# 196 "/usr/include/sys/param.h" 3 4

/*
 * btodb() is messy and perhaps slow because `bytes' may be an off_t.  We
 * want to shift an unsigned type to avoid sign extension and we don't
 * want to widen `bytes' unnecessarily.  Assume that the result fits in
 * a daddr_t.
 */
#ifndef btodb
#define btodb(bytes)	 		/* calculates (bytes / DEV_BSIZE) */ \
	(sizeof (bytes) > sizeof(long) \
	 ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \
	 : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT))
#endif
# 209 "/usr/include/sys/param.h" 3 4

#ifndef dbtob
#define dbtob(db)			/* calculates (db * DEV_BSIZE) */ \
	((off_t)(db) << DEV_BSHIFT)
#endif
# 214 "/usr/include/sys/param.h" 3 4

#define	PRIMASK	0x0ff
#define	PCATCH	0x100		/* OR'd with pri for tsleep to check signals */
#define	PDROP	0x200	/* OR'd with pri to stop re-entry of interlock mutex */

#define	NZERO	0		/* default "nice" */

#define	NBBY	8		/* number of bits in a byte */
#define	NBPW	sizeof(int)	/* number of bytes per word (integer) */

#define	CMASK	022		/* default file mask: S_IWGRP|S_IWOTH */

#define	NODEV	(dev_t)(-1)	/* non-existent device */

/*
 * File system parameters and macros.
 *
 * MAXBSIZE -	Filesystems are made out of blocks of at most MAXBSIZE bytes
 *		per block.  MAXBSIZE may be made larger without effecting
 *		any existing filesystems as long as it does not exceed MAXPHYS,
 *		and may be made smaller at the risk of not being able to use
 *		filesystems which require a block size exceeding MAXBSIZE.
 *
 * MAXBCACHEBUF - Maximum size of a buffer in the buffer cache.  This must
 *		be >= MAXBSIZE and can be set differently for different
 *		architectures by defining it in <machine/param.h>.
 *		Making this larger allows NFS to do larger reads/writes.
 *
 * BKVASIZE -	Nominal buffer space per buffer, in bytes.  BKVASIZE is the
 *		minimum KVM memory reservation the kernel is willing to make.
 *		Filesystems can of course request smaller chunks.  Actual 
 *		backing memory uses a chunk size of a page (PAGE_SIZE).
 *		The default value here can be overridden on a per-architecture
 *		basis by defining it in <machine/param.h>.  This should
 *		probably be done to increase its value, when MAXBCACHEBUF is
 *		defined as a larger value in <machine/param.h>.
 *
 *		If you make BKVASIZE too small you risk seriously fragmenting
 *		the buffer KVM map which may slow things down a bit.  If you
 *		make it too big the kernel will not be able to optimally use 
 *		the KVM memory reserved for the buffer cache and will wind 
 *		up with too-few buffers.
 *
 *		The default is 16384, roughly 2x the block size used by a
 *		normal UFS filesystem.
 */
#define MAXBSIZE	65536	/* must be power of 2 */
#ifndef	MAXBCACHEBUF
#define	MAXBCACHEBUF	MAXBSIZE /* must be a power of 2 >= MAXBSIZE */
#endif
# 264 "/usr/include/sys/param.h" 3 4
#ifndef	BKVASIZE
#define BKVASIZE	16384	/* must be power of 2 */
#endif
# 267 "/usr/include/sys/param.h" 3 4
#define BKVAMASK	(BKVASIZE-1)

/*
 * MAXPATHLEN defines the longest permissible path length after expanding
 * symbolic links. It is used to allocate a temporary buffer from the buffer
 * pool in which to do the name expansion, hence should be a power of two,
 * and must be less than or equal to MAXBSIZE.  MAXSYMLINKS defines the
 * maximum number of symbolic links that may be expanded in a path name.
 * It should be set high enough to allow all legitimate uses, but halt
 * infinite loops reasonably quickly.
 */
#define	MAXPATHLEN	PATH_MAX
#define MAXSYMLINKS	32

/* Bit map related macros. */
#define	setbit(a,i)	(((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY))
#define	clrbit(a,i)	(((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY)))
#define	isset(a,i)							\
	(((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY)))
#define	isclr(a,i)							\
	((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0)

/* Macros for counting and rounding. */
#ifndef howmany
#define	howmany(x, y)	(((x)+((y)-1))/(y))
#endif
# 293 "/usr/include/sys/param.h" 3 4
#define	nitems(x)	(sizeof((x)) / sizeof((x)[0]))
#define	rounddown(x, y)	(((x)/(y))*(y))
#define	rounddown2(x, y) ((x)&(~((y)-1)))          /* if y is power of two */
#define	roundup(x, y)	((((x)+((y)-1))/(y))*(y))  /* to any y */
#define	roundup2(x, y)	(((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */
#define powerof2(x)	((((x)-1)&(x))==0)

/* Macros for min/max. */
#define	MIN(a,b) (((a)<(b))?(a):(b))
#define	MAX(a,b) (((a)>(b))?(a):(b))

#ifdef _KERNEL
/*
 * Basic byte order function prototypes for non-inline functions.
 */
#ifndef LOCORE
#ifndef _BYTEORDER_PROTOTYPED
#define	_BYTEORDER_PROTOTYPED
__BEGIN_DECLS
__uint32_t	 htonl(__uint32_t);
__uint16_t	 htons(__uint16_t);
__uint32_t	 ntohl(__uint32_t);
__uint16_t	 ntohs(__uint16_t);
__END_DECLS
#endif
# 318 "/usr/include/sys/param.h" 3 4
#endif
# 319 "/usr/include/sys/param.h" 3 4

#ifndef lint
#ifndef _BYTEORDER_FUNC_DEFINED
#define	_BYTEORDER_FUNC_DEFINED
#define	htonl(x)	__htonl(x)
#define	htons(x)	__htons(x)
#define	ntohl(x)	__ntohl(x)
#define	ntohs(x)	__ntohs(x)
#endif /* !_BYTEORDER_FUNC_DEFINED */
# 328 "/usr/include/sys/param.h" 3 4
#endif /* lint */
# 329 "/usr/include/sys/param.h" 3 4
#endif /* _KERNEL */
# 330 "/usr/include/sys/param.h" 3 4

/*
 * Scale factor for scaled integers used to count %cpu time and load avgs.
 *
 * The number of CPU `tick's that map to a unique `%age' can be expressed
 * by the formula (1 / (2 ^ (FSHIFT - 11))).  The maximum load average that
 * can be calculated (assuming 32 bits) can be closely approximated using
 * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15).
 *
 * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age',
 * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024.
 */
#define	FSHIFT	11		/* bits to right of fixed binary point */
#define FSCALE	(1<<FSHIFT)

#define dbtoc(db)			/* calculates devblks to pages */ \
	((db + (ctodb(1) - 1)) >> (PAGE_SHIFT - DEV_BSHIFT))
 
#define ctodb(db)			/* calculates pages to devblks */ \
	((db) << (PAGE_SHIFT - DEV_BSHIFT))

/*
 * Old spelling of __containerof().
 */
#define	member2struct(s, m, x)						\
	((struct s *)(void *)((char *)(x) - offsetof(struct s, m)))

/*
 * Access a variable length array that has been declared as a fixed
 * length array.
 */
#define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset])

#endif	/* _SYS_PARAM_H_ */
# 364 "/usr/include/sys/param.h" 3 4
# 46 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 46 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/fcntl.h" 1 3 4
/*-
 * Copyright (c) 1983, 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)fcntl.h	8.3 (Berkeley) 1/21/94
 * $FreeBSD: head/sys/sys/fcntl.h 294205 2016-01-17 08:40:51Z kib $
 */

#ifndef _SYS_FCNTL_H_
#define	_SYS_FCNTL_H_

/*
 * This file includes the definitions for open and fcntl
 * described by POSIX for <fcntl.h>; it also includes
 * related kernel definitions.
 */

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 47 "/usr/include/fcntl.h" 3 4
# 48 "/usr/include/fcntl.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 48 "/usr/include/fcntl.h" 3 4
# 49 "/usr/include/fcntl.h" 3 4

#ifndef _MODE_T_DECLARED
typedef	__mode_t	mode_t;
#define	_MODE_T_DECLARED
#endif
# 54 "/usr/include/fcntl.h" 3 4

#ifndef _OFF_T_DECLARED
typedef	__off_t		off_t;
#define	_OFF_T_DECLARED
#endif
# 59 "/usr/include/fcntl.h" 3 4

#ifndef _PID_T_DECLARED
typedef	__pid_t		pid_t;
#define	_PID_T_DECLARED
#endif
# 64 "/usr/include/fcntl.h" 3 4

/*
 * File status flags: these are used by open(2), fcntl(2).
 * They are also used (indirectly) in the kernel file structure f_flags,
 * which is a superset of the open/fcntl flags.  Open flags and f_flags
 * are inter-convertible using OFLAGS(fflags) and FFLAGS(oflags).
 * Open/fcntl flags begin with O_; kernel-internal flags begin with F.
 */
/* open-only flags */
#define	O_RDONLY	0x0000		/* open for reading only */
#define	O_WRONLY	0x0001		/* open for writing only */
#define	O_RDWR		0x0002		/* open for reading and writing */
#define	O_ACCMODE	0x0003		/* mask for above modes */

/*
 * Kernel encoding of open mode; separate read and write bits that are
 * independently testable: 1 greater than the above.
 *
 * XXX
 * FREAD and FWRITE are excluded from the #ifdef _KERNEL so that TIOCFLUSH,
 * which was documented to use FREAD/FWRITE, continues to work.
 */
#if __BSD_VISIBLE
#define	FREAD		0x0001
#define	FWRITE		0x0002
#endif
# 90 "/usr/include/fcntl.h" 3 4
#define	O_NONBLOCK	0x0004		/* no delay */
#define	O_APPEND	0x0008		/* set append mode */
#if __BSD_VISIBLE
#define	O_SHLOCK	0x0010		/* open with shared file lock */
#define	O_EXLOCK	0x0020		/* open with exclusive file lock */
#define	O_ASYNC		0x0040		/* signal pgrp when data ready */
#define	O_FSYNC		0x0080		/* synchronous writes */
#endif
# 98 "/usr/include/fcntl.h" 3 4
#define	O_SYNC		0x0080		/* POSIX synonym for O_FSYNC */
#if __POSIX_VISIBLE >= 200809
#define	O_NOFOLLOW	0x0100		/* don't follow symlinks */
#endif
# 102 "/usr/include/fcntl.h" 3 4
#define	O_CREAT		0x0200		/* create if nonexistent */
#define	O_TRUNC		0x0400		/* truncate to zero length */
#define	O_EXCL		0x0800		/* error if already exists */
#ifdef _KERNEL
#define	FHASLOCK	0x4000		/* descriptor holds advisory lock */
#endif
# 108 "/usr/include/fcntl.h" 3 4

/* Defined by POSIX 1003.1; BSD default, but must be distinct from O_RDONLY. */
#define	O_NOCTTY	0x8000		/* don't assign controlling terminal */

#if __BSD_VISIBLE
/* Attempt to bypass buffer cache */
#define	O_DIRECT	0x00010000
#endif
# 116 "/usr/include/fcntl.h" 3 4

#if __POSIX_VISIBLE >= 200809
#define	O_DIRECTORY	0x00020000	/* Fail if not directory */
#define	O_EXEC		0x00040000	/* Open for execute only */
#endif
# 121 "/usr/include/fcntl.h" 3 4
#ifdef	_KERNEL
#define	FEXEC		O_EXEC
#endif
# 124 "/usr/include/fcntl.h" 3 4

#if __POSIX_VISIBLE >= 200809
/* Defined by POSIX 1003.1-2008; BSD default, but reserve for future use. */
#define	O_TTY_INIT	0x00080000	/* Restore default termios attributes */

#define	O_CLOEXEC	0x00100000
#endif
# 131 "/usr/include/fcntl.h" 3 4

#if __BSD_VISIBLE
#define	O_VERIFY	0x00200000	/* open only after verification */
#endif
# 135 "/usr/include/fcntl.h" 3 4

/*
 * XXX missing O_DSYNC, O_RSYNC.
 */

#ifdef _KERNEL

/* Only for devfs d_close() flags. */
#define	FLASTCLOSE	O_DIRECTORY
#define	FREVOKE		O_VERIFY
/* Only for fo_close() from half-succeeded open */
#define	FOPENFAILED	O_TTY_INIT

/* convert from open() flags to/from fflags; convert O_RD/WR to FREAD/FWRITE */
#define	FFLAGS(oflags)	((oflags) & O_EXEC ? (oflags) : (oflags) + 1)
#define	OFLAGS(fflags)	((fflags) & O_EXEC ? (fflags) : (fflags) - 1)

/* bits to save after open */
#define	FMASK	(FREAD|FWRITE|FAPPEND|FASYNC|FFSYNC|FNONBLOCK|O_DIRECT|FEXEC)
/* bits settable by fcntl(F_SETFL, ...) */
#define	FCNTLFLAGS	(FAPPEND|FASYNC|FFSYNC|FNONBLOCK|FRDAHEAD|O_DIRECT)

#if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \
    defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4)
/*
 * Set by shm_open(3) in older libc's to get automatic MAP_ASYNC
 * behavior for POSIX shared memory objects (which are otherwise
 * implemented as plain files).
 */
#define	FPOSIXSHM	O_NOFOLLOW
#undef FCNTLFLAGS
#define	FCNTLFLAGS	(FAPPEND|FASYNC|FFSYNC|FNONBLOCK|FPOSIXSHM|FRDAHEAD| \
			 O_DIRECT)
#endif
# 169 "/usr/include/fcntl.h" 3 4
#endif
# 170 "/usr/include/fcntl.h" 3 4

/*
 * The O_* flags used to have only F* names, which were used in the kernel
 * and by fcntl.  We retain the F* names for the kernel f_flag field
 * and for backward compatibility for fcntl.  These flags are deprecated.
 */
#if __BSD_VISIBLE
#define	FAPPEND		O_APPEND	/* kernel/compat */
#define	FASYNC		O_ASYNC		/* kernel/compat */
#define	FFSYNC		O_FSYNC		/* kernel */
#define	FNONBLOCK	O_NONBLOCK	/* kernel */
#define	FNDELAY		O_NONBLOCK	/* compat */
#define	O_NDELAY	O_NONBLOCK	/* compat */
#endif
# 184 "/usr/include/fcntl.h" 3 4

/*
 * We are out of bits in f_flag (which is a short).  However,
 * the flag bits not set in FMASK are only meaningful in the
 * initial open syscall.  Those bits can thus be given a
 * different meaning for fcntl(2).
 */
#if __BSD_VISIBLE
/* Read ahead */
#define	FRDAHEAD	O_CREAT
#endif
# 195 "/usr/include/fcntl.h" 3 4

#if __POSIX_VISIBLE >= 200809
/*
 * Magic value that specify the use of the current working directory
 * to determine the target of relative file paths in the openat() and
 * similar syscalls.
 */
#define	AT_FDCWD		-100

/*
 * Miscellaneous flags for the *at() syscalls.
 */
#define	AT_EACCESS		0x100	/* Check access using effective user and group ID */
#define	AT_SYMLINK_NOFOLLOW	0x200   /* Do not follow symbolic links */
#define	AT_SYMLINK_FOLLOW	0x400	/* Follow symbolic link */
#define	AT_REMOVEDIR		0x800	/* Remove directory instead of file */
#endif
# 212 "/usr/include/fcntl.h" 3 4

/*
 * Constants used for fcntl(2)
 */

/* command values */
#define	F_DUPFD		0		/* duplicate file descriptor */
#define	F_GETFD		1		/* get file descriptor flags */
#define	F_SETFD		2		/* set file descriptor flags */
#define	F_GETFL		3		/* get file status flags */
#define	F_SETFL		4		/* set file status flags */
#if __XSI_VISIBLE || __POSIX_VISIBLE >= 200112
#define	F_GETOWN	5		/* get SIGIO/SIGURG proc/pgrp */
#define	F_SETOWN	6		/* set SIGIO/SIGURG proc/pgrp */
#endif
# 227 "/usr/include/fcntl.h" 3 4
#if __BSD_VISIBLE
#define	F_OGETLK	7		/* get record locking information */
#define	F_OSETLK	8		/* set record locking information */
#define	F_OSETLKW	9		/* F_SETLK; wait if blocked */
#define	F_DUP2FD	10		/* duplicate file descriptor to arg */
#endif
# 233 "/usr/include/fcntl.h" 3 4
#define	F_GETLK		11		/* get record locking information */
#define	F_SETLK		12		/* set record locking information */
#define	F_SETLKW	13		/* F_SETLK; wait if blocked */
#if __BSD_VISIBLE
#define	F_SETLK_REMOTE	14		/* debugging support for remote locks */
#define	F_READAHEAD	15		/* read ahead */
#define	F_RDAHEAD	16		/* Darwin compatible read ahead */
#endif
# 241 "/usr/include/fcntl.h" 3 4
#if __POSIX_VISIBLE >= 200809
#define	F_DUPFD_CLOEXEC	17		/* Like F_DUPFD, but FD_CLOEXEC is set */
#endif
# 244 "/usr/include/fcntl.h" 3 4
#if __BSD_VISIBLE
#define	F_DUP2FD_CLOEXEC 18		/* Like F_DUP2FD, but FD_CLOEXEC is set */
#endif
# 247 "/usr/include/fcntl.h" 3 4

/* file descriptor flags (F_GETFD, F_SETFD) */
#define	FD_CLOEXEC	1		/* close-on-exec flag */

/* record locking flags (F_GETLK, F_SETLK, F_SETLKW) */
#define	F_RDLCK		1		/* shared or read lock */
#define	F_UNLCK		2		/* unlock */
#define	F_WRLCK		3		/* exclusive or write lock */
#if __BSD_VISIBLE
#define	F_UNLCKSYS	4		/* purge locks for a given system ID */ 
#define	F_CANCEL	5		/* cancel an async lock request */
#endif
# 259 "/usr/include/fcntl.h" 3 4
#ifdef _KERNEL
#define	F_WAIT		0x010		/* Wait until lock is granted */
#define	F_FLOCK		0x020	 	/* Use flock(2) semantics for lock */
#define	F_POSIX		0x040	 	/* Use POSIX semantics for lock */
#define	F_REMOTE	0x080		/* Lock owner is remote NFS client */
#define	F_NOINTR	0x100		/* Ignore signals when waiting */
#endif
# 266 "/usr/include/fcntl.h" 3 4

/*
 * Advisory file segment locking data type -
 * information passed to system by user
 */
struct flock {
	off_t	l_start;	/* starting offset */
	off_t	l_len;		/* len = 0 means until end of file */
	pid_t	l_pid;		/* lock owner */
	short	l_type;		/* lock type: read/write, etc. */
	short	l_whence;	/* type of l_start */
	int	l_sysid;	/* remote system id or zero for local */
};

#if __BSD_VISIBLE
/*
 * Old advisory file segment locking data type,
 * before adding l_sysid.
 */
struct __oflock {
	off_t	l_start;	/* starting offset */
	off_t	l_len;		/* len = 0 means until end of file */
	pid_t	l_pid;		/* lock owner */
	short	l_type;		/* lock type: read/write, etc. */
	short	l_whence;	/* type of l_start */
};
#endif
# 293 "/usr/include/fcntl.h" 3 4

#if __BSD_VISIBLE
/* lock operations for flock(2) */
#define	LOCK_SH		0x01		/* shared file lock */
#define	LOCK_EX		0x02		/* exclusive file lock */
#define	LOCK_NB		0x04		/* don't block when locking */
#define	LOCK_UN		0x08		/* unlock file */
#endif
# 301 "/usr/include/fcntl.h" 3 4

#if __POSIX_VISIBLE >= 200112
/*
 * Advice to posix_fadvise
 */
#define	POSIX_FADV_NORMAL	0	/* no special treatment */
#define	POSIX_FADV_RANDOM	1	/* expect random page references */
#define	POSIX_FADV_SEQUENTIAL	2	/* expect sequential page references */
#define	POSIX_FADV_WILLNEED	3	/* will need these pages */
#define	POSIX_FADV_DONTNEED	4	/* dont need these pages */
#define	POSIX_FADV_NOREUSE	5	/* access data only once */
#endif
# 313 "/usr/include/fcntl.h" 3 4

#ifndef _KERNEL
__BEGIN_DECLS
int	open(const char *, int, ...);
int	creat(const char *, mode_t);
int	fcntl(int, int, ...);
#if __BSD_VISIBLE
int	flock(int, int);
#endif
# 322 "/usr/include/fcntl.h" 3 4
#if __POSIX_VISIBLE >= 200809
int	openat(int, const char *, int, ...);
#endif
# 325 "/usr/include/fcntl.h" 3 4
#if __POSIX_VISIBLE >= 200112
int	posix_fadvise(int, off_t, off_t, int);
int	posix_fallocate(int, off_t, off_t);
#endif
# 329 "/usr/include/fcntl.h" 3 4
__END_DECLS
#endif
# 331 "/usr/include/fcntl.h" 3 4

#endif /* !_SYS_FCNTL_H_ */
# 333 "/usr/include/fcntl.h" 3 4
# 47 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_errno.h"
#endif /* expanded by -frewrite-includes */
# 47 "sys/unix/erl_unix_sys.h"
# 1 "../include/internal/erl_errno.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2009. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef ERL_ERRNO_H__
#define ERL_ERRNO_H__

#if 0 /* expanded by -frewrite-includes */
#include <errno.h>
#endif /* expanded by -frewrite-includes */
# 24 "../include/internal/erl_errno.h"
# 1 "/usr/include/errno.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)errno.h	8.5 (Berkeley) 1/21/94
 * $FreeBSD: head/sys/sys/errno.h 296715 2016-03-12 07:54:42Z trasz $
 */

#ifndef _SYS_ERRNO_H_
#define _SYS_ERRNO_H_

#ifndef _KERNEL
#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 42 "/usr/include/errno.h" 3 4
# 43 "/usr/include/errno.h" 3 4
__BEGIN_DECLS
int *	__error(void);
__END_DECLS
#define	errno		(* __error())
#endif
# 48 "/usr/include/errno.h" 3 4

#define	EPERM		1		/* Operation not permitted */
#define	ENOENT		2		/* No such file or directory */
#define	ESRCH		3		/* No such process */
#define	EINTR		4		/* Interrupted system call */
#define	EIO		5		/* Input/output error */
#define	ENXIO		6		/* Device not configured */
#define	E2BIG		7		/* Argument list too long */
#define	ENOEXEC		8		/* Exec format error */
#define	EBADF		9		/* Bad file descriptor */
#define	ECHILD		10		/* No child processes */
#define	EDEADLK		11		/* Resource deadlock avoided */
					/* 11 was EAGAIN */
#define	ENOMEM		12		/* Cannot allocate memory */
#define	EACCES		13		/* Permission denied */
#define	EFAULT		14		/* Bad address */
#ifndef _POSIX_SOURCE
#define	ENOTBLK		15		/* Block device required */
#endif
# 67 "/usr/include/errno.h" 3 4
#define	EBUSY		16		/* Device busy */
#define	EEXIST		17		/* File exists */
#define	EXDEV		18		/* Cross-device link */
#define	ENODEV		19		/* Operation not supported by device */
#define	ENOTDIR		20		/* Not a directory */
#define	EISDIR		21		/* Is a directory */
#define	EINVAL		22		/* Invalid argument */
#define	ENFILE		23		/* Too many open files in system */
#define	EMFILE		24		/* Too many open files */
#define	ENOTTY		25		/* Inappropriate ioctl for device */
#ifndef _POSIX_SOURCE
#define	ETXTBSY		26		/* Text file busy */
#endif
# 80 "/usr/include/errno.h" 3 4
#define	EFBIG		27		/* File too large */
#define	ENOSPC		28		/* No space left on device */
#define	ESPIPE		29		/* Illegal seek */
#define	EROFS		30		/* Read-only filesystem */
#define	EMLINK		31		/* Too many links */
#define	EPIPE		32		/* Broken pipe */

/* math software */
#define	EDOM		33		/* Numerical argument out of domain */
#define	ERANGE		34		/* Result too large */

/* non-blocking and interrupt i/o */
#define	EAGAIN		35		/* Resource temporarily unavailable */
#ifndef _POSIX_SOURCE
#define	EWOULDBLOCK	EAGAIN		/* Operation would block */
#define	EINPROGRESS	36		/* Operation now in progress */
#define	EALREADY	37		/* Operation already in progress */

/* ipc/network software -- argument errors */
#define	ENOTSOCK	38		/* Socket operation on non-socket */
#define	EDESTADDRREQ	39		/* Destination address required */
#define	EMSGSIZE	40		/* Message too long */
#define	EPROTOTYPE	41		/* Protocol wrong type for socket */
#define	ENOPROTOOPT	42		/* Protocol not available */
#define	EPROTONOSUPPORT	43		/* Protocol not supported */
#define	ESOCKTNOSUPPORT	44		/* Socket type not supported */
#define	EOPNOTSUPP	45		/* Operation not supported */
#define	ENOTSUP		EOPNOTSUPP	/* Operation not supported */
#define	EPFNOSUPPORT	46		/* Protocol family not supported */
#define	EAFNOSUPPORT	47		/* Address family not supported by protocol family */
#define	EADDRINUSE	48		/* Address already in use */
#define	EADDRNOTAVAIL	49		/* Can't assign requested address */

/* ipc/network software -- operational errors */
#define	ENETDOWN	50		/* Network is down */
#define	ENETUNREACH	51		/* Network is unreachable */
#define	ENETRESET	52		/* Network dropped connection on reset */
#define	ECONNABORTED	53		/* Software caused connection abort */
#define	ECONNRESET	54		/* Connection reset by peer */
#define	ENOBUFS		55		/* No buffer space available */
#define	EISCONN		56		/* Socket is already connected */
#define	ENOTCONN	57		/* Socket is not connected */
#define	ESHUTDOWN	58		/* Can't send after socket shutdown */
#define	ETOOMANYREFS	59		/* Too many references: can't splice */
#define	ETIMEDOUT	60		/* Operation timed out */
#define	ECONNREFUSED	61		/* Connection refused */

#define	ELOOP		62		/* Too many levels of symbolic links */
#endif /* _POSIX_SOURCE */
# 129 "/usr/include/errno.h" 3 4
#define	ENAMETOOLONG	63		/* File name too long */

/* should be rearranged */
#ifndef _POSIX_SOURCE
#define	EHOSTDOWN	64		/* Host is down */
#define	EHOSTUNREACH	65		/* No route to host */
#endif /* _POSIX_SOURCE */
# 136 "/usr/include/errno.h" 3 4
#define	ENOTEMPTY	66		/* Directory not empty */

/* quotas & mush */
#ifndef _POSIX_SOURCE
#define	EPROCLIM	67		/* Too many processes */
#define	EUSERS		68		/* Too many users */
#define	EDQUOT		69		/* Disc quota exceeded */

/* Network File System */
#define	ESTALE		70		/* Stale NFS file handle */
#define	EREMOTE		71		/* Too many levels of remote in path */
#define	EBADRPC		72		/* RPC struct is bad */
#define	ERPCMISMATCH	73		/* RPC version wrong */
#define	EPROGUNAVAIL	74		/* RPC prog. not avail */
#define	EPROGMISMATCH	75		/* Program version wrong */
#define	EPROCUNAVAIL	76		/* Bad procedure for program */
#endif /* _POSIX_SOURCE */
# 153 "/usr/include/errno.h" 3 4

#define	ENOLCK		77		/* No locks available */
#define	ENOSYS		78		/* Function not implemented */

#ifndef _POSIX_SOURCE
#define	EFTYPE		79		/* Inappropriate file type or format */
#define	EAUTH		80		/* Authentication error */
#define	ENEEDAUTH	81		/* Need authenticator */
#define	EIDRM		82		/* Identifier removed */
#define	ENOMSG		83		/* No message of desired type */
#define	EOVERFLOW	84		/* Value too large to be stored in data type */
#define	ECANCELED	85		/* Operation canceled */
#define	EILSEQ		86		/* Illegal byte sequence */
#define	ENOATTR		87		/* Attribute not found */

#define	EDOOFUS		88		/* Programming error */
#endif /* _POSIX_SOURCE */
# 170 "/usr/include/errno.h" 3 4

#define	EBADMSG		89		/* Bad message */
#define	EMULTIHOP	90		/* Multihop attempted */
#define	ENOLINK		91		/* Link has been severed */
#define	EPROTO		92		/* Protocol error */

#ifndef _POSIX_SOURCE
#define	ENOTCAPABLE	93		/* Capabilities insufficient */
#define	ECAPMODE	94		/* Not permitted in capability mode */
#define	ENOTRECOVERABLE	95		/* State not recoverable */
#define	EOWNERDEAD	96		/* Previous owner died */
#endif /* _POSIX_SOURCE */
# 182 "/usr/include/errno.h" 3 4

#ifndef _POSIX_SOURCE
#define	ELAST		96		/* Must be equal largest errno */
#endif /* _POSIX_SOURCE */
# 186 "/usr/include/errno.h" 3 4

#ifdef _KERNEL
/* pseudo-errors returned inside kernel to modify return to process */
#define	ERESTART	(-1)		/* restart syscall */
#define	EJUSTRETURN	(-2)		/* don't modify regs, just return */
#define	ENOIOCTL	(-3)		/* ioctl not handled by this layer */
#define	EDIRIOCTL	(-4)		/* do direct ioctl in GEOM */
#define	ERELOOKUP	(-5)		/* retry the directory lookup */
#endif
# 195 "/usr/include/errno.h" 3 4

#endif
# 197 "/usr/include/errno.h" 3 4
# 25 "../include/internal/erl_errno.h" 2
/*
 * Make sure that ENOTSUP is defined.
 */
#ifndef ENOTSUP
#  ifdef EOPNOTSUPP
#    define ENOTSUP EOPNOTSUPP
#else
# 32 "../include/internal/erl_errno.h"
#    define ENOTSUP INT_MAX
#  endif
# 34 "../include/internal/erl_errno.h"
#endif
# 35 "../include/internal/erl_errno.h"

#ifdef __WIN32__
#  ifndef EWOULDBLOCK
#    define EWOULDBLOCK (10035) /* WSAEWOULDBLOCK */
#  endif
# 40 "../include/internal/erl_errno.h"
#  ifndef ETIMEDOUT
#    define ETIMEDOUT (10060) /* WSAETIMEDOUT */
#  endif
# 43 "../include/internal/erl_errno.h"
#else
# 44 "../include/internal/erl_errno.h"
#  ifndef EWOULDBLOCK
#    define EWOULDBLOCK EAGAIN
#  endif
# 47 "../include/internal/erl_errno.h"
#  ifndef ETIMEDOUT
#    define ETIMEDOUT EAGAIN
#  endif
# 50 "../include/internal/erl_errno.h"
#endif
# 51 "../include/internal/erl_errno.h"

#endif
# 53 "../include/internal/erl_errno.h"
# 48 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <signal.h>
#endif /* expanded by -frewrite-includes */
# 48 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/signal.h" 1 3 4
/*-
 * Copyright (c) 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)signal.h	8.3 (Berkeley) 3/30/94
 * $FreeBSD: head/include/signal.h 295602 2016-02-13 17:23:55Z kib $
 */

#ifndef _SIGNAL_H_
#define	_SIGNAL_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/signal.h" 3 4
# 37 "/usr/include/signal.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/signal.h" 3 4
# 38 "/usr/include/signal.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/signal.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/signal.h" 3 4
# 39 "/usr/include/signal.h" 3 4
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#if 0 /* expanded by -frewrite-includes */
#include <machine/ucontext.h>
#endif /* expanded by -frewrite-includes */
# 40 "/usr/include/signal.h" 3 4
# 1 "/usr/include/machine/ucontext.h" 1 3 4
/*	$NetBSD: mcontext.h,v 1.4 2003/10/08 22:43:01 thorpej Exp $	*/

/*-
 * Copyright (c) 2001, 2002 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Klaus Klein and by Jason R. Thorpe of Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/arm/include/ucontext.h 266570 2014-05-23 00:21:02Z imp $
 */

#ifndef _MACHINE_MCONTEXT_H_
#define _MACHINE_MCONTEXT_H_
/*
 * General register state
 */
#define _NGREG		17
typedef unsigned int	__greg_t;
typedef __greg_t	__gregset_t[_NGREG];

#define _REG_R0		0
#define _REG_R1		1
#define _REG_R2		2
#define _REG_R3		3
#define _REG_R4		4
#define _REG_R5		5
#define _REG_R6		6
#define _REG_R7		7
#define _REG_R8		8
#define _REG_R9		9
#define _REG_R10	10
#define _REG_R11	11
#define _REG_R12	12
#define _REG_R13	13
#define _REG_R14	14
#define _REG_R15	15
#define _REG_CPSR	16
/* Convenience synonyms */
#define _REG_FP		_REG_R11
#define _REG_SP		_REG_R13
#define _REG_LR		_REG_R14
#define _REG_PC		_REG_R15

/*
 * Floating point register state
 */
/* Note: the storage layout of this structure must be identical to ARMFPE! */
typedef struct {
	unsigned int	__fp_fpsr;
	struct {
		unsigned int	__fp_exponent;
		unsigned int	__fp_mantissa_hi;
		unsigned int	__fp_mantissa_lo;
	}		__fp_fr[8];
} __fpregset_t;

typedef struct {
	unsigned int	__vfp_fpscr;
	unsigned int	__vfp_fstmx[33];
	unsigned int	__vfp_fpsid;
} __vfpregset_t;

typedef struct {
	__gregset_t	__gregs;
	union {
		__fpregset_t __fpregs;
		__vfpregset_t __vfpregs;
	} __fpu;
} mcontext_t;

/* Machine-dependent uc_flags */
#define	_UC_ARM_VFP	0x00010000	/* FPU field is VFP */

/* used by signal delivery to indicate status of signal stack */
#define _UC_SETSTACK	0x00020000
#define _UC_CLRSTACK	0x00040000

#endif	/* !_MACHINE_MCONTEXT_H_ */
# 101 "/usr/include/machine/ucontext.h" 3 4
# 41 "/usr/include/signal.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_ucontext.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/signal.h" 3 4
# 1 "/usr/include/sys/_ucontext.h" 1 3 4
/*-
 * Copyright (c) 1999 Marcel Moolenaar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer 
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/_ucontext.h 295561 2016-02-12 07:38:19Z kib $
 */

#ifndef _SYS__UCONTEXT_H_
#define	_SYS__UCONTEXT_H_

typedef struct __ucontext {
	/*
	 * Keep the order of the first two fields. Also,
	 * keep them the first two fields in the structure.
	 * This way we can have a union with struct
	 * sigcontext and ucontext_t. This allows us to
	 * support them both at the same time.
	 * note: the union is not defined, though.
	 */
	__sigset_t	uc_sigmask;
	mcontext_t	uc_mcontext;

	struct __ucontext *uc_link;
	struct __stack_t uc_stack;
	int		uc_flags;
	int		__spare__[4];
} ucontext_t;

#endif /* _SYS__UCONTEXT_H */
# 53 "/usr/include/sys/_ucontext.h" 3 4
# 42 "/usr/include/signal.h" 2 3 4
#endif
# 43 "/usr/include/signal.h" 3 4

#if __BSD_VISIBLE
/*
 * XXX should enlarge these, if only to give empty names instead of bounds
 * errors for large signal numbers.
 */
extern const char * const sys_signame[NSIG];
extern const char * const sys_siglist[NSIG];
extern const int sys_nsig;
#endif
# 53 "/usr/include/signal.h" 3 4

#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
#ifndef _PID_T_DECLARED
typedef	__pid_t		pid_t;
#define	_PID_T_DECLARED
#endif
# 59 "/usr/include/signal.h" 3 4
#endif
# 60 "/usr/include/signal.h" 3 4

#if __POSIX_VISIBLE || __XSI_VISIBLE
struct pthread;		/* XXX */
typedef struct pthread *__pthread_t;
#if !defined(_PTHREAD_T_DECLARED) && __POSIX_VISIBLE >= 200809
typedef __pthread_t pthread_t;
#define	_PTHREAD_T_DECLARED
#endif
# 68 "/usr/include/signal.h" 3 4
#endif /* __POSIX_VISIBLE || __XSI_VISIBLE */
# 69 "/usr/include/signal.h" 3 4

__BEGIN_DECLS
int	raise(int);

#if __POSIX_VISIBLE || __XSI_VISIBLE
int	kill(__pid_t, int);
int	pthread_kill(__pthread_t, int);
int	pthread_sigmask(int, const __sigset_t * __restrict,
	    __sigset_t * __restrict);
int	sigaction(int, const struct sigaction * __restrict,
	    struct sigaction * __restrict);
int	sigaddset(sigset_t *, int);
int	sigdelset(sigset_t *, int);
int	sigemptyset(sigset_t *);
int	sigfillset(sigset_t *);
int	sigismember(const sigset_t *, int);
int	sigpending(sigset_t *) __nonnull(1);
int	sigprocmask(int, const sigset_t * __restrict, sigset_t * __restrict);
int	sigsuspend(const sigset_t *) __nonnull(1);
int	sigwait(const sigset_t * __restrict, int * __restrict) __nonnull_all;
#endif
# 90 "/usr/include/signal.h" 3 4

#if __POSIX_VISIBLE >= 199506 || __XSI_VISIBLE >= 600
int	sigqueue(__pid_t, int, const union sigval);

struct timespec;
int	sigtimedwait(const sigset_t * __restrict, siginfo_t * __restrict,
	    const struct timespec * __restrict);
int	sigwaitinfo(const sigset_t * __restrict, siginfo_t * __restrict);
#endif
# 99 "/usr/include/signal.h" 3 4

#if __XSI_VISIBLE
int	killpg(__pid_t, int);
int	sigaltstack(const stack_t * __restrict, stack_t * __restrict); 
int	sighold(int);
int	sigignore(int);
int	sigpause(int);
int	sigrelse(int);
void	(*sigset(int, void (*)(int)))(int);
int	xsi_sigpause(int);
#endif
# 110 "/usr/include/signal.h" 3 4

#if __XSI_VISIBLE >= 600
int	siginterrupt(int, int);
#endif
# 114 "/usr/include/signal.h" 3 4

#if __POSIX_VISIBLE >= 200809
void	psignal(unsigned int, const char *);
#endif
# 118 "/usr/include/signal.h" 3 4

#if __BSD_VISIBLE
int	sigblock(int);
int	sigreturn(const struct __ucontext *);
int	sigsetmask(int);
int	sigstack(const struct sigstack *, struct sigstack *);
int	sigvec(int, struct sigvec *, struct sigvec *);
#endif
# 126 "/usr/include/signal.h" 3 4
__END_DECLS

#endif /* !_SIGNAL_H_ */
# 129 "/usr/include/signal.h" 3 4
# 49 "sys/unix/erl_unix_sys.h" 2
#if 0 /* expanded by -frewrite-includes */
#include <setjmp.h>
#endif /* expanded by -frewrite-includes */
# 49 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/setjmp.h" 1 3 4
/*-
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)setjmp.h	8.2 (Berkeley) 1/21/94
 * $FreeBSD: head/include/setjmp.h 265878 2014-05-11 13:48:21Z jilles $
 */

#ifndef _SETJMP_H_
#define _SETJMP_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/setjmp.h" 3 4
# 42 "/usr/include/setjmp.h" 3 4

/* The size of the jmp_buf is machine dependent: */
#if 0 /* expanded by -frewrite-includes */
#include <machine/setjmp.h>
#endif /* expanded by -frewrite-includes */
# 44 "/usr/include/setjmp.h" 3 4
# 1 "/usr/include/machine/setjmp.h" 1 3 4
/*	$NetBSD: setjmp.h,v 1.5 2013/01/11 13:56:32 matt Exp $	*/
/* $FreeBSD: head/sys/arm/include/setjmp.h 251517 2013-06-08 07:16:22Z andrew $ */

/*
 * machine/setjmp.h: machine dependent setjmp-related information.
 */

#ifndef _MACHINE_SETJMP_H_
#define _MACHINE_SETJMP_H_

#define	_JBLEN	64		/* size, in longs, of a jmp_buf */

/*
 * NOTE: The internal structure of a jmp_buf is *PRIVATE*
 *       This information is provided as there is software
 *       that fiddles with this with obtain the stack pointer
 *	 (yes really ! and its commercial !).
 *
 * Description of the setjmp buffer
 *
 * word  0	magic number	(dependent on creator)
 *	13	fpscr		vfp status control register
 *	14	r4		register 4
 *	15	r5		register 5
 *	16	r6		register 6
 *	17	r7		register 7
 *	18	r8		register 8
 *	19	r9		register 9
 *	20	r10		register 10 (sl)
 *	21	r11		register 11 (fp)
 *	22	r12		register 12 (ip)
 *	23	r13		register 13 (sp)
 *	24	r14		register 14 (lr)
 *	25	signal mask	(dependent on magic)
 *	26	(con't)
 *	27	(con't)
 *	28	(con't)
 *	32-33	d8		(vfp register d8)
 *	34-35	d9		(vfp register d9)
 *	36-37	d10		(vfp register d10)
 *	38-39	d11		(vfp register d11)
 *	40-41	d12		(vfp register d12)
 *	42-43	d13		(vfp register d13)
 *	44-45	d14		(vfp register d14)
 *	46-47	d15		(vfp register d15)
 *
 * The magic number number identifies the jmp_buf and
 * how the buffer was created as well as providing
 * a sanity check
 *
 * A side note I should mention - Please do not tamper
 * with the floating point fields. While they are
 * always saved and restored at the moment this cannot
 * be garenteed especially if the compiler happens
 * to be generating soft-float code so no fp
 * registers will be used.
 *
 * Whilst this can be seen an encouraging people to
 * use the setjmp buffer in this way I think that it
 * is for the best then if changes occur compiles will
 * break rather than just having new builds falling over
 * mysteriously.
 */

#define _JB_MAGIC__SETJMP	0x4278f500
#define _JB_MAGIC_SETJMP	0x4278f501
#define _JB_MAGIC__SETJMP_VFP	0x4278f502
#define _JB_MAGIC_SETJMP_VFP	0x4278f503

/* Valid for all jmp_buf's */

#define _JB_MAGIC		 0
#define _JB_REG_FPSCR		13
#define _JB_REG_R4		14
#define _JB_REG_R5		15
#define _JB_REG_R6		16
#define _JB_REG_R7		17
#define _JB_REG_R8		18
#define _JB_REG_R9		19
#define _JB_REG_R10		20
#define _JB_REG_R11		21
#define _JB_REG_R12		22
#define _JB_REG_R13		23
#define _JB_REG_R14		24

/* Only valid with the _JB_MAGIC_SETJMP magic */

#define _JB_SIGMASK		25

#define	_JB_REG_D8		32
#define	_JB_REG_D9		34
#define	_JB_REG_D10		36
#define	_JB_REG_D11		38
#define	_JB_REG_D12		40
#define	_JB_REG_D13		42
#define	_JB_REG_D14		44
#define	_JB_REG_D15		46

#ifndef __ASSEMBLER__
#if __BSD_VISIBLE || __POSIX_VISIBLE || __XSI_VISIBLE
typedef struct _sigjmp_buf { int _sjb[_JBLEN + 1]; } sigjmp_buf[1];
#endif
# 103 "/usr/include/machine/setjmp.h" 3 4

typedef struct _jmp_buf { int _jb[_JBLEN + 1]; } jmp_buf[1];
#endif
# 106 "/usr/include/machine/setjmp.h" 3 4

#endif /* !_MACHINE_SETJMP_H_ */
# 108 "/usr/include/machine/setjmp.h" 3 4
# 45 "/usr/include/setjmp.h" 2 3 4

__BEGIN_DECLS
#if __XSI_VISIBLE >= 600
void	_longjmp(jmp_buf, int) __dead2;
int	_setjmp(jmp_buf) __returns_twice;
#endif
# 51 "/usr/include/setjmp.h" 3 4
void	longjmp(jmp_buf, int) __dead2;
#if __BSD_VISIBLE
void	longjmperror(void);
#endif
# 55 "/usr/include/setjmp.h" 3 4
int	setjmp(jmp_buf) __returns_twice;
#if __POSIX_VISIBLE || __XSI_VISIBLE
void	siglongjmp(sigjmp_buf, int) __dead2;
int	sigsetjmp(sigjmp_buf, int) __returns_twice;
#endif
# 60 "/usr/include/setjmp.h" 3 4
__END_DECLS

#endif /* !_SETJMP_H_ */
# 63 "/usr/include/setjmp.h" 3 4
# 50 "sys/unix/erl_unix_sys.h" 2

#if HAVE_SYS_SOCKETIO_H
#if 0 /* expanded by -frewrite-includes */
#   include <sys/socketio.h>
#endif /* expanded by -frewrite-includes */
# 52 "sys/unix/erl_unix_sys.h"
# 53 "sys/unix/erl_unix_sys.h"
#endif
# 54 "sys/unix/erl_unix_sys.h"
#if HAVE_SYS_SOCKIO_H
#if 0 /* expanded by -frewrite-includes */
#   include <sys/sockio.h>
#endif /* expanded by -frewrite-includes */
# 55 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/sys/sockio.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1990, 1993, 1994
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)sockio.h	8.1 (Berkeley) 3/28/94
 * $FreeBSD: head/sys/sys/sockio.h 283364 2015-05-24 11:24:14Z melifaro $
 */

#ifndef _SYS_SOCKIO_H_
#define	_SYS_SOCKIO_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/ioccom.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/sockio.h" 3 4
# 1 "/usr/include/sys/ioccom.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1990, 1993, 1994
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)ioccom.h	8.2 (Berkeley) 3/28/94
 * $FreeBSD: head/sys/sys/ioccom.h 206051 2010-04-01 16:21:35Z pjd $
 */

#ifndef	_SYS_IOCCOM_H_
#define	_SYS_IOCCOM_H_

/*
 * Ioctl's have the command encoded in the lower word, and the size of
 * any in or out parameters in the upper word.  The high 3 bits of the
 * upper word are used to encode the in/out status of the parameter.
 */
#define	IOCPARM_SHIFT	13		/* number of bits for ioctl size */
#define	IOCPARM_MASK	((1 << IOCPARM_SHIFT) - 1) /* parameter length mask */
#define	IOCPARM_LEN(x)	(((x) >> 16) & IOCPARM_MASK)
#define	IOCBASECMD(x)	((x) & ~(IOCPARM_MASK << 16))
#define	IOCGROUP(x)	(((x) >> 8) & 0xff)

#define	IOCPARM_MAX	(1 << IOCPARM_SHIFT) /* max size of ioctl */
#define	IOC_VOID	0x20000000	/* no parameters */
#define	IOC_OUT		0x40000000	/* copy out parameters */
#define	IOC_IN		0x80000000	/* copy in parameters */
#define	IOC_INOUT	(IOC_IN|IOC_OUT)
#define	IOC_DIRMASK	(IOC_VOID|IOC_OUT|IOC_IN)

#define	_IOC(inout,group,num,len)	((unsigned long) \
	((inout) | (((len) & IOCPARM_MASK) << 16) | ((group) << 8) | (num)))
#define	_IO(g,n)	_IOC(IOC_VOID,	(g), (n), 0)
#define	_IOWINT(g,n)	_IOC(IOC_VOID,	(g), (n), sizeof(int))
#define	_IOR(g,n,t)	_IOC(IOC_OUT,	(g), (n), sizeof(t))
#define	_IOW(g,n,t)	_IOC(IOC_IN,	(g), (n), sizeof(t))
/* this should be _IORW, but stdio got there first */
#define	_IOWR(g,n,t)	_IOC(IOC_INOUT,	(g), (n), sizeof(t))

#ifdef _KERNEL

#if defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD5) || \
    defined(COMPAT_FREEBSD4) || defined(COMPAT_43)
#define	IOCPARM_IVAL(x)	((int)(intptr_t)(void *)*(caddr_t *)(void *)(x))
#endif
# 69 "/usr/include/sys/ioccom.h" 3 4

#else
# 71 "/usr/include/sys/ioccom.h" 3 4

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 72 "/usr/include/sys/ioccom.h" 3 4
# 73 "/usr/include/sys/ioccom.h" 3 4

__BEGIN_DECLS
int	ioctl(int, unsigned long, ...);
__END_DECLS

#endif
# 79 "/usr/include/sys/ioccom.h" 3 4

#endif /* !_SYS_IOCCOM_H_ */
# 81 "/usr/include/sys/ioccom.h" 3 4
# 37 "/usr/include/sys/sockio.h" 2 3 4

/* Socket ioctl's. */
#define	SIOCSHIWAT	 _IOW('s',  0, int)		/* set high watermark */
#define	SIOCGHIWAT	 _IOR('s',  1, int)		/* get high watermark */
#define	SIOCSLOWAT	 _IOW('s',  2, int)		/* set low watermark */
#define	SIOCGLOWAT	 _IOR('s',  3, int)		/* get low watermark */
#define	SIOCATMARK	 _IOR('s',  7, int)		/* at oob mark? */
#define	SIOCSPGRP	 _IOW('s',  8, int)		/* set process group */
#define	SIOCGPGRP	 _IOR('s',  9, int)		/* get process group */

/*	SIOCADDRT	 _IOW('r', 10, struct ortentry)	4.3BSD */
/*	SIOCDELRT	 _IOW('r', 11, struct ortentry)	4.3BSD */
#define	SIOCGETVIFCNT	_IOWR('r', 15, struct sioc_vif_req)/* get vif pkt cnt */
#define	SIOCGETSGCNT	_IOWR('r', 16, struct sioc_sg_req) /* get s,g pkt cnt */

#define	SIOCSIFADDR	 _IOW('i', 12, struct ifreq)	/* set ifnet address */
/*	OSIOCGIFADDR	_IOWR('i', 13, struct ifreq)	4.3BSD */
#define	SIOCGIFADDR	_IOWR('i', 33, struct ifreq)	/* get ifnet address */
#define	SIOCSIFDSTADDR	 _IOW('i', 14, struct ifreq)	/* set p-p address */
/*	OSIOCGIFDSTADDR	_IOWR('i', 15, struct ifreq)	4.3BSD */
#define	SIOCGIFDSTADDR	_IOWR('i', 34, struct ifreq)	/* get p-p address */
#define	SIOCSIFFLAGS	 _IOW('i', 16, struct ifreq)	/* set ifnet flags */
#define	SIOCGIFFLAGS	_IOWR('i', 17, struct ifreq)	/* get ifnet flags */
/*	OSIOCGIFBRDADDR	_IOWR('i', 18, struct ifreq)	4.3BSD */
#define	SIOCGIFBRDADDR	_IOWR('i', 35, struct ifreq)	/* get broadcast addr */
#define	SIOCSIFBRDADDR	 _IOW('i', 19, struct ifreq)	/* set broadcast addr */
/*	OSIOCGIFCONF	_IOWR('i', 20, struct ifconf)	4.3BSD */
#define	SIOCGIFCONF	_IOWR('i', 36, struct ifconf)	/* get ifnet list */
/*	OSIOCGIFNETMASK	_IOWR('i', 21, struct ifreq)	4.3BSD */
#define	SIOCGIFNETMASK	_IOWR('i', 37, struct ifreq)	/* get net addr mask */
#define	SIOCSIFNETMASK	 _IOW('i', 22, struct ifreq)	/* set net addr mask */
#define	SIOCGIFMETRIC	_IOWR('i', 23, struct ifreq)	/* get IF metric */
#define	SIOCSIFMETRIC	 _IOW('i', 24, struct ifreq)	/* set IF metric */
#define	SIOCDIFADDR	 _IOW('i', 25, struct ifreq)	/* delete IF addr */
#define	OSIOCAIFADDR	 _IOW('i', 26, struct oifaliasreq) /* FreeBSD 9.x */
/*	SIOCALIFADDR	 _IOW('i', 27, struct if_laddrreq) KAME */
/*	SIOCGLIFADDR	_IOWR('i', 28, struct if_laddrreq) KAME */
/*	SIOCDLIFADDR	 _IOW('i', 29, struct if_laddrreq) KAME */
#define	SIOCSIFCAP	 _IOW('i', 30, struct ifreq)	/* set IF features */
#define	SIOCGIFCAP	_IOWR('i', 31, struct ifreq)	/* get IF features */
#define	SIOCGIFINDEX	_IOWR('i', 32, struct ifreq)	/* get IF index */
#define	SIOCGIFMAC	_IOWR('i', 38, struct ifreq)	/* get IF MAC label */
#define	SIOCSIFMAC	 _IOW('i', 39, struct ifreq)	/* set IF MAC label */
#define	SIOCSIFNAME	 _IOW('i', 40, struct ifreq)	/* set IF name */
#define	SIOCSIFDESCR	 _IOW('i', 41, struct ifreq)	/* set ifnet descr */ 
#define	SIOCGIFDESCR	_IOWR('i', 42, struct ifreq)	/* get ifnet descr */ 
#define	SIOCAIFADDR	 _IOW('i', 43, struct ifaliasreq)/* add/chg IF alias */

#define	SIOCADDMULTI	 _IOW('i', 49, struct ifreq)	/* add m'cast addr */
#define	SIOCDELMULTI	 _IOW('i', 50, struct ifreq)	/* del m'cast addr */
#define	SIOCGIFMTU	_IOWR('i', 51, struct ifreq)	/* get IF mtu */
#define	SIOCSIFMTU	 _IOW('i', 52, struct ifreq)	/* set IF mtu */
#define	SIOCGIFPHYS	_IOWR('i', 53, struct ifreq)	/* get IF wire */
#define	SIOCSIFPHYS	 _IOW('i', 54, struct ifreq)	/* set IF wire */
#define	SIOCSIFMEDIA	_IOWR('i', 55, struct ifreq)	/* set net media */
#define	SIOCGIFMEDIA	_IOWR('i', 56, struct ifmediareq) /* get net media */

#define	SIOCSIFGENERIC	 _IOW('i', 57, struct ifreq)	/* generic IF set op */
#define	SIOCGIFGENERIC	_IOWR('i', 58, struct ifreq)	/* generic IF get op */

#define	SIOCGIFSTATUS	_IOWR('i', 59, struct ifstat)	/* get IF status */
#define	SIOCSIFLLADDR	 _IOW('i', 60, struct ifreq)	/* set linklevel addr */
#define	SIOCGI2C	_IOWR('i', 61, struct ifreq)	/* get I2C data  */

#define	SIOCSIFPHYADDR	 _IOW('i', 70, struct ifaliasreq) /* set gif addres */
#define	SIOCGIFPSRCADDR	_IOWR('i', 71, struct ifreq)	/* get gif psrc addr */
#define	SIOCGIFPDSTADDR	_IOWR('i', 72, struct ifreq)	/* get gif pdst addr */
#define	SIOCDIFPHYADDR	 _IOW('i', 73, struct ifreq)	/* delete gif addrs */
/*	SIOCSLIFPHYADDR	 _IOW('i', 74, struct if_laddrreq) KAME */
/*	SIOCGLIFPHYADDR	_IOWR('i', 75, struct if_laddrreq) KAME */

#define	SIOCGPRIVATE_0	_IOWR('i', 80, struct ifreq)	/* device private 0 */
#define	SIOCGPRIVATE_1	_IOWR('i', 81, struct ifreq)	/* device private 1 */

#define	SIOCSIFVNET	_IOWR('i', 90, struct ifreq)	/* move IF jail/vnet */
#define	SIOCSIFRVNET	_IOWR('i', 91, struct ifreq)	/* reclaim vnet IF */

#define	SIOCGIFFIB	_IOWR('i', 92, struct ifreq)	/* get IF fib */
#define	SIOCSIFFIB	 _IOW('i', 93, struct ifreq)	/* set IF fib */

#define	SIOCGTUNFIB	_IOWR('i', 94, struct ifreq)	/* get tunnel fib */
#define	SIOCSTUNFIB	 _IOW('i', 95, struct ifreq)	/* set tunnel fib */

#define	SIOCSDRVSPEC	_IOW('i', 123, struct ifdrv)	/* set driver-specific
								  parameters */
#define	SIOCGDRVSPEC	_IOWR('i', 123, struct ifdrv)	/* get driver-specific
								  parameters */

#define	SIOCIFCREATE	_IOWR('i', 122, struct ifreq)	/* create clone if */
#define	SIOCIFCREATE2	_IOWR('i', 124, struct ifreq)	/* create clone if */
#define	SIOCIFDESTROY	 _IOW('i', 121, struct ifreq)	/* destroy clone if */
#define	SIOCIFGCLONERS	_IOWR('i', 120, struct if_clonereq) /* get cloners */

#define	SIOCAIFGROUP	 _IOW('i', 135, struct ifgroupreq) /* add an ifgroup */
#define	SIOCGIFGROUP	_IOWR('i', 136, struct ifgroupreq) /* get ifgroups */
#define	SIOCDIFGROUP	 _IOW('i', 137, struct ifgroupreq) /* delete ifgroup */
#define	SIOCGIFGMEMB	_IOWR('i', 138, struct ifgroupreq) /* get members */
#define	SIOCGIFXMEDIA	_IOWR('i', 139, struct ifmediareq) /* get net xmedia */

#endif /* !_SYS_SOCKIO_H_ */
# 137 "/usr/include/sys/sockio.h" 3 4
# 56 "sys/unix/erl_unix_sys.h" 2
#endif
# 57 "sys/unix/erl_unix_sys.h"

#ifdef HAVE_NET_ERRNO_H
#if 0 /* expanded by -frewrite-includes */
#include <net/errno.h>
#endif /* expanded by -frewrite-includes */
# 59 "sys/unix/erl_unix_sys.h"
# 60 "sys/unix/erl_unix_sys.h"
#endif
# 61 "sys/unix/erl_unix_sys.h"

#ifdef HAVE_DIRENT_H
#if 0 /* expanded by -frewrite-includes */
#  include <dirent.h>
#endif /* expanded by -frewrite-includes */
# 63 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/dirent.h" 1 3 4
/*-
 * Copyright (c) 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)dirent.h	8.2 (Berkeley) 7/28/94
 * $FreeBSD: head/include/dirent.h 268531 2014-07-11 16:16:26Z jhb $
 */

#ifndef _DIRENT_H_
#define _DIRENT_H_

/*
 * The kernel defines the format of directory entries returned by
 * the getdirentries(2) system call.
 */
#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 40 "/usr/include/dirent.h" 3 4
# 41 "/usr/include/dirent.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/dirent.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/dirent.h" 3 4
# 1 "/usr/include/sys/dirent.h" 1 3 4
/*-
 * Copyright (c) 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)dirent.h	8.3 (Berkeley) 8/10/94
 * $FreeBSD: head/sys/sys/dirent.h 161347 2006-08-16 09:34:56Z maxim $
 */

#ifndef	_SYS_DIRENT_H_
#define	_SYS_DIRENT_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/dirent.h" 3 4
# 37 "/usr/include/sys/dirent.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/sys/dirent.h" 3 4
# 38 "/usr/include/sys/dirent.h" 3 4

/*
 * The dirent structure defines the format of directory entries returned by
 * the getdirentries(2) system call.
 *
 * A directory entry has a struct dirent at the front of it, containing its
 * inode number, the length of the entry, and the length of the name
 * contained in the entry.  These are followed by the name padded to a 4
 * byte boundary with null bytes.  All names are guaranteed null terminated.
 * The maximum length of a name in a directory is MAXNAMLEN.
 */

struct dirent {
	__uint32_t d_fileno;		/* file number of entry */
	__uint16_t d_reclen;		/* length of this record */
	__uint8_t  d_type; 		/* file type, see below */
	__uint8_t  d_namlen;		/* length of string in d_name */
#if __BSD_VISIBLE
#define	MAXNAMLEN	255
	char	d_name[MAXNAMLEN + 1];	/* name must be no longer than this */
#else
# 59 "/usr/include/sys/dirent.h" 3 4
	char	d_name[255 + 1];	/* name must be no longer than this */
#endif
# 61 "/usr/include/sys/dirent.h" 3 4
};

#if __BSD_VISIBLE
/*
 * File types
 */
#define	DT_UNKNOWN	 0
#define	DT_FIFO		 1
#define	DT_CHR		 2
#define	DT_DIR		 4
#define	DT_BLK		 6
#define	DT_REG		 8
#define	DT_LNK		10
#define	DT_SOCK		12
#define	DT_WHT		14

/*
 * Convert between stat structure types and directory types.
 */
#define	IFTODT(mode)	(((mode) & 0170000) >> 12)
#define	DTTOIF(dirtype)	((dirtype) << 12)

/*
 * The _GENERIC_DIRSIZ macro gives the minimum record length which will hold
 * the directory entry.  This returns the amount of space in struct direct
 * without the d_name field, plus enough space for the name with a terminating
 * null byte (dp->d_namlen+1), rounded up to a 4 byte boundary.
 *
 * XXX although this macro is in the implementation namespace, it requires
 * a manifest constant that is not.
 */
#define	_GENERIC_DIRSIZ(dp) \
    ((sizeof (struct dirent) - (MAXNAMLEN+1)) + (((dp)->d_namlen+1 + 3) &~ 3))
#endif /* __BSD_VISIBLE */
# 95 "/usr/include/sys/dirent.h" 3 4

#ifdef _KERNEL
#define	GENERIC_DIRSIZ(dp)	_GENERIC_DIRSIZ(dp)
#endif
# 99 "/usr/include/sys/dirent.h" 3 4

#endif /* !_SYS_DIRENT_H_ */
# 101 "/usr/include/sys/dirent.h" 3 4
# 42 "/usr/include/dirent.h" 2 3 4

#if __XSI_VISIBLE
/*
 * XXX this is probably illegal in the __XSI_VISIBLE case, but brings us closer
 * to the specification.
 */
#define	d_ino		d_fileno	/* backward and XSI compatibility */
#endif
# 50 "/usr/include/dirent.h" 3 4

#if __BSD_VISIBLE

#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 53 "/usr/include/dirent.h" 3 4
# 54 "/usr/include/dirent.h" 3 4

/* definitions for library routines operating on directories. */
#define	DIRBLKSIZ	1024

struct _dirdesc;
typedef struct _dirdesc DIR;

/* flags for opendir2 */
#define DTF_HIDEW	0x0001	/* hide whiteout entries */
#define DTF_NODUP	0x0002	/* don't return duplicate names */
#define DTF_REWIND	0x0004	/* rewind after reading union stack */
#define __DTF_READALL	0x0008	/* everything has been read */
#define	__DTF_SKIPREAD	0x0010  /* assume internal buffer is populated */

#else /* !__BSD_VISIBLE */
# 69 "/usr/include/dirent.h" 3 4

typedef	void *	DIR;

#endif /* __BSD_VISIBLE */
# 73 "/usr/include/dirent.h" 3 4

#ifndef _KERNEL

__BEGIN_DECLS
#if __POSIX_VISIBLE >= 200809 || __XSI_VISIBLE >= 700
int	 alphasort(const struct dirent **, const struct dirent **);
int	 dirfd(DIR *);
#endif
# 81 "/usr/include/dirent.h" 3 4
#if __BSD_VISIBLE
DIR	*__opendir2(const char *, int);
int	 fdclosedir(DIR *);
int	 getdents(int, char *, int);
int	 getdirentries(int, char *, int, long *);
#endif
# 87 "/usr/include/dirent.h" 3 4
DIR	*opendir(const char *);
DIR	*fdopendir(int);
struct dirent *
	 readdir(DIR *);
#if __POSIX_VISIBLE >= 199506 || __XSI_VISIBLE >= 500
int	 readdir_r(DIR *, struct dirent *, struct dirent **);
#endif
# 94 "/usr/include/dirent.h" 3 4
void	 rewinddir(DIR *);
#if __POSIX_VISIBLE >= 200809 || __XSI_VISIBLE >= 700
int	 scandir(const char *, struct dirent ***,
	    int (*)(const struct dirent *), int (*)(const struct dirent **,
	    const struct dirent **));
#ifdef __BLOCKS__
int	 scandir_b(const char *, struct dirent ***,
	    int (^)(const struct dirent *),
	    int (^)(const struct dirent **, const struct dirent **));
#endif
# 104 "/usr/include/dirent.h" 3 4
#endif
# 105 "/usr/include/dirent.h" 3 4
#if __XSI_VISIBLE
void	 seekdir(DIR *, long);
long	 telldir(DIR *);
#endif
# 109 "/usr/include/dirent.h" 3 4
int	 closedir(DIR *);
__END_DECLS

#endif /* !_KERNEL */
# 113 "/usr/include/dirent.h" 3 4

#endif /* !_DIRENT_H_ */
# 115 "/usr/include/dirent.h" 3 4
# 64 "sys/unix/erl_unix_sys.h" 2
#endif
# 65 "sys/unix/erl_unix_sys.h"

#ifdef HAVE_UNISTD_H
#if 0 /* expanded by -frewrite-includes */
#   include <unistd.h>
#endif /* expanded by -frewrite-includes */
# 67 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/unistd.h" 1 3 4
/*-
 * Copyright (c) 1991, 1993, 1994
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)unistd.h	8.12 (Berkeley) 4/27/95
 * $FreeBSD: head/include/unistd.h 296162 2016-02-28 17:52:33Z kib $
 */

#ifndef _UNISTD_H_
#define	_UNISTD_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/unistd.h" 3 4
# 37 "/usr/include/unistd.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/types.h>			/* XXX adds too much pollution. */
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/unistd.h" 3 4
# 38 "/usr/include/unistd.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/unistd.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/unistd.h" 3 4
# 1 "/usr/include/sys/unistd.h" 1 3 4
/*-
 * Copyright (c) 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)unistd.h	8.2 (Berkeley) 1/7/94
 * $FreeBSD: head/sys/sys/unistd.h 296572 2016-03-09 19:05:11Z jhb $
 */

#ifndef _SYS_UNISTD_H_
#define	_SYS_UNISTD_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/unistd.h" 3 4
# 37 "/usr/include/sys/unistd.h" 3 4

/*
 * POSIX options and option groups we unconditionally do or don't
 * implement.  Those options which are implemented (or not) entirely
 * in user mode are defined in <unistd.h>.  Please keep this list in
 * alphabetical order.
 *
 * Anything which is defined as zero below **must** have an
 * implementation for the corresponding sysconf() which is able to
 * determine conclusively whether or not the feature is supported.
 * Anything which is defined as other than -1 below **must** have
 * complete headers, types, and function declarations as specified by
 * the POSIX standard; however, if the relevant sysconf() function
 * returns -1, the functions may be stubbed out.
 */
#define	_POSIX_ADVISORY_INFO		200112L
#define	_POSIX_ASYNCHRONOUS_IO		200112L
#define	_POSIX_CHOWN_RESTRICTED		1
#define	_POSIX_CLOCK_SELECTION		(-1)
#define	_POSIX_CPUTIME			200112L
#define	_POSIX_FSYNC			200112L
#define	_POSIX_IPV6			0
#define	_POSIX_JOB_CONTROL		1
#define	_POSIX_MAPPED_FILES		200112L
#define	_POSIX_MEMLOCK			(-1)
#define	_POSIX_MEMLOCK_RANGE		200112L
#define	_POSIX_MEMORY_PROTECTION	200112L
#define	_POSIX_MESSAGE_PASSING		200112L
#define	_POSIX_MONOTONIC_CLOCK		200112L
#define	_POSIX_NO_TRUNC			1
#define	_POSIX_PRIORITIZED_IO		(-1)
#define	_POSIX_PRIORITY_SCHEDULING	200112L
#define	_POSIX_RAW_SOCKETS		200112L
#define	_POSIX_REALTIME_SIGNALS		200112L
#define	_POSIX_SEMAPHORES		200112L
#define	_POSIX_SHARED_MEMORY_OBJECTS	200112L
#define	_POSIX_SPORADIC_SERVER		(-1)
#define	_POSIX_SYNCHRONIZED_IO		(-1)
#define	_POSIX_TIMEOUTS			200112L
#define	_POSIX_TIMERS			200112L
#define	_POSIX_TYPED_MEMORY_OBJECTS	(-1)
#define	_POSIX_VDISABLE			0xff

#if __XSI_VISIBLE
#define	_XOPEN_SHM			1
#define	_XOPEN_STREAMS			(-1)
#endif
# 84 "/usr/include/sys/unistd.h" 3 4

/*
 * Although we have saved user/group IDs, we do not use them in setuid
 * as described in POSIX 1003.1, because the feature does not work for
 * root.  We use the saved IDs in seteuid/setegid, which are not currently
 * part of the POSIX 1003.1 specification.  XXX revisit for 1003.1-2001
 * as this is now mandatory.
 */
#ifdef	_NOT_AVAILABLE
#define	_POSIX_SAVED_IDS	1 /* saved set-user-ID and set-group-ID */
#endif
# 95 "/usr/include/sys/unistd.h" 3 4

/* Define the POSIX.1 version we target for compliance. */
#define	_POSIX_VERSION		200112L

/* access function */
#define	F_OK		0	/* test for existence of file */
#define	X_OK		0x01	/* test for execute or search permission */
#define	W_OK		0x02	/* test for write permission */
#define	R_OK		0x04	/* test for read permission */

/* whence values for lseek(2) */
#ifndef SEEK_SET
#define	SEEK_SET	0	/* set file offset to offset */
#define	SEEK_CUR	1	/* set file offset to current plus offset */
#define	SEEK_END	2	/* set file offset to EOF plus offset */
#endif
# 111 "/usr/include/sys/unistd.h" 3 4
#if __BSD_VISIBLE
#define	SEEK_DATA	3	/* set file offset to next data past offset */
#define	SEEK_HOLE	4	/* set file offset to next hole past offset */
#endif
# 115 "/usr/include/sys/unistd.h" 3 4

#ifndef _POSIX_SOURCE
/* whence values for lseek(2); renamed by POSIX 1003.1 */
#define	L_SET		SEEK_SET
#define	L_INCR		SEEK_CUR
#define	L_XTND		SEEK_END
#endif
# 122 "/usr/include/sys/unistd.h" 3 4

/* configurable pathname variables */
#define	_PC_LINK_MAX		 1
#define	_PC_MAX_CANON		 2
#define	_PC_MAX_INPUT		 3
#define	_PC_NAME_MAX		 4
#define	_PC_PATH_MAX		 5
#define	_PC_PIPE_BUF		 6
#define	_PC_CHOWN_RESTRICTED	 7
#define	_PC_NO_TRUNC		 8
#define	_PC_VDISABLE		 9

#if __POSIX_VISIBLE >= 199309
#define	_PC_ASYNC_IO		53
#define	_PC_PRIO_IO		54
#define	_PC_SYNC_IO		55
#endif
# 139 "/usr/include/sys/unistd.h" 3 4

#if __POSIX_VISIBLE >= 200112
#define	_PC_ALLOC_SIZE_MIN	10
#define	_PC_FILESIZEBITS	12
#define	_PC_REC_INCR_XFER_SIZE	14
#define	_PC_REC_MAX_XFER_SIZE	15
#define	_PC_REC_MIN_XFER_SIZE	16
#define	_PC_REC_XFER_ALIGN	17
#define	_PC_SYMLINK_MAX		18
#endif
# 149 "/usr/include/sys/unistd.h" 3 4

#if __BSD_VISIBLE
#define	_PC_ACL_EXTENDED	59
#define	_PC_ACL_PATH_MAX	60
#define	_PC_CAP_PRESENT		61
#define	_PC_INF_PRESENT		62
#define	_PC_MAC_PRESENT		63
#define	_PC_ACL_NFS4		64
#endif
# 158 "/usr/include/sys/unistd.h" 3 4

/* From OpenSolaris, used by SEEK_DATA/SEEK_HOLE. */
#define	_PC_MIN_HOLE_SIZE	21

#if __BSD_VISIBLE
/*
 * rfork() options.
 *
 * XXX currently, some operations without RFPROC set are not supported.
 */
#define	RFNAMEG		(1<<0)	/* UNIMPL new plan9 `name space' */
#define	RFENVG		(1<<1)	/* UNIMPL copy plan9 `env space' */
#define	RFFDG		(1<<2)	/* copy fd table */
#define	RFNOTEG		(1<<3)	/* UNIMPL create new plan9 `note group' */
#define	RFPROC		(1<<4)	/* change child (else changes curproc) */
#define	RFMEM		(1<<5)	/* share `address space' */
#define	RFNOWAIT	(1<<6)	/* give child to init */
#define	RFCNAMEG	(1<<10)	/* UNIMPL zero plan9 `name space' */
#define	RFCENVG		(1<<11)	/* UNIMPL zero plan9 `env space' */
#define	RFCFDG		(1<<12)	/* close all fds, zero fd table */
#define	RFTHREAD	(1<<13)	/* enable kernel thread support */
#define	RFSIGSHARE	(1<<14)	/* share signal handlers */
#define	RFLINUXTHPN	(1<<16)	/* do linux clone exit parent notification */
#define	RFSTOPPED	(1<<17)	/* leave child in a stopped state */
#define	RFHIGHPID	(1<<18)	/* use a pid higher than 10 (idleproc) */
#define	RFTSIGZMB	(1<<19)	/* select signal for exit parent notification */
#define	RFTSIGSHIFT	20	/* selected signal number is in bits 20-27  */
#define	RFTSIGMASK	0xFF
#define	RFTSIGNUM(flags)	(((flags) >> RFTSIGSHIFT) & RFTSIGMASK)
#define	RFTSIGFLAGS(signum)	((signum) << RFTSIGSHIFT)
#define	RFPROCDESC	(1<<28)	/* return a process descriptor */
#define	RFPPWAIT	(1<<31)	/* parent sleeps until child exits (vfork) */
#define	RFFLAGS		(RFFDG | RFPROC | RFMEM | RFNOWAIT | RFCFDG | \
    RFTHREAD | RFSIGSHARE | RFLINUXTHPN | RFSTOPPED | RFHIGHPID | RFTSIGZMB | \
    RFPROCDESC | RFPPWAIT)
#define	RFKERNELONLY	(RFSTOPPED | RFHIGHPID | RFPPWAIT | RFPROCDESC)

#endif /* __BSD_VISIBLE */
# 196 "/usr/include/sys/unistd.h" 3 4

#endif /* !_SYS_UNISTD_H_ */
# 198 "/usr/include/sys/unistd.h" 3 4
# 39 "/usr/include/unistd.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_null.h>
#endif /* expanded by -frewrite-includes */
# 39 "/usr/include/unistd.h" 3 4
# 40 "/usr/include/unistd.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 40 "/usr/include/unistd.h" 3 4
# 41 "/usr/include/unistd.h" 3 4

#ifndef _GID_T_DECLARED
typedef	__gid_t		gid_t;
#define	_GID_T_DECLARED
#endif
# 46 "/usr/include/unistd.h" 3 4

#ifndef _OFF_T_DECLARED
typedef	__off_t		off_t;
#define	_OFF_T_DECLARED
#endif
# 51 "/usr/include/unistd.h" 3 4

#ifndef _PID_T_DECLARED
typedef	__pid_t		pid_t;
#define	_PID_T_DECLARED
#endif
# 56 "/usr/include/unistd.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 61 "/usr/include/unistd.h" 3 4

#ifndef _SSIZE_T_DECLARED
typedef	__ssize_t	ssize_t;
#define	_SSIZE_T_DECLARED
#endif
# 66 "/usr/include/unistd.h" 3 4

#ifndef _UID_T_DECLARED
typedef	__uid_t		uid_t;
#define	_UID_T_DECLARED
#endif
# 71 "/usr/include/unistd.h" 3 4

#ifndef _USECONDS_T_DECLARED
typedef	__useconds_t	useconds_t;
#define	_USECONDS_T_DECLARED
#endif
# 76 "/usr/include/unistd.h" 3 4

#define	STDIN_FILENO	0	/* standard input file descriptor */
#define	STDOUT_FILENO	1	/* standard output file descriptor */
#define	STDERR_FILENO	2	/* standard error file descriptor */

#if __XSI_VISIBLE || __POSIX_VISIBLE >= 200112
#define	F_ULOCK		0	/* unlock locked section */
#define	F_LOCK		1	/* lock a section for exclusive use */
#define	F_TLOCK		2	/* test and lock a section for exclusive use */
#define	F_TEST		3	/* test a section for locks by other procs */
#endif
# 87 "/usr/include/unistd.h" 3 4

/*
 * POSIX options and option groups we unconditionally do or don't
 * implement.  This list includes those options which are exclusively
 * implemented (or not) in user mode.  Please keep this list in
 * alphabetical order.
 *
 * Anything which is defined as zero below **must** have an
 * implementation for the corresponding sysconf() which is able to
 * determine conclusively whether or not the feature is supported.
 * Anything which is defined as other than -1 below **must** have
 * complete headers, types, and function declarations as specified by
 * the POSIX standard; however, if the relevant sysconf() function
 * returns -1, the functions may be stubbed out.
 */
#define	_POSIX_BARRIERS			200112L
#define	_POSIX_CPUTIME			200112L
#define	_POSIX_READER_WRITER_LOCKS	200112L
#define	_POSIX_REGEXP			1
#define	_POSIX_SHELL			1
#define	_POSIX_SPAWN			200112L
#define	_POSIX_SPIN_LOCKS		200112L
#define	_POSIX_THREAD_ATTR_STACKADDR	200112L
#define	_POSIX_THREAD_ATTR_STACKSIZE	200112L
#define	_POSIX_THREAD_CPUTIME		200112L
#define	_POSIX_THREAD_PRIO_INHERIT	200112L
#define	_POSIX_THREAD_PRIO_PROTECT	200112L
#define	_POSIX_THREAD_PRIORITY_SCHEDULING 200112L
#define	_POSIX_THREAD_PROCESS_SHARED	200112L
#define	_POSIX_THREAD_SAFE_FUNCTIONS	-1
#define	_POSIX_THREAD_SPORADIC_SERVER	-1
#define	_POSIX_THREADS			200112L
#define	_POSIX_TRACE			-1
#define	_POSIX_TRACE_EVENT_FILTER	-1
#define	_POSIX_TRACE_INHERIT		-1
#define	_POSIX_TRACE_LOG		-1
#define	_POSIX2_C_BIND			200112L	/* mandatory */
#define	_POSIX2_C_DEV			-1 /* need c99 utility */
#define	_POSIX2_CHAR_TERM		1
#define	_POSIX2_FORT_DEV		-1 /* need fort77 utility */
#define	_POSIX2_FORT_RUN		200112L
#define	_POSIX2_LOCALEDEF		-1
#define	_POSIX2_PBS			-1
#define	_POSIX2_PBS_ACCOUNTING		-1
#define	_POSIX2_PBS_CHECKPOINT		-1
#define	_POSIX2_PBS_LOCATE		-1
#define	_POSIX2_PBS_MESSAGE		-1
#define	_POSIX2_PBS_TRACK		-1
#define	_POSIX2_SW_DEV			-1 /* XXX ??? */
#define	_POSIX2_UPE			200112L
#define	_V6_ILP32_OFF32			-1
#define	_V6_ILP32_OFFBIG		0
#define	_V6_LP64_OFF64			0
#define	_V6_LPBIG_OFFBIG		-1

#if __XSI_VISIBLE
#define	_XOPEN_CRYPT			-1 /* XXX ??? */
#define	_XOPEN_ENH_I18N			-1 /* mandatory in XSI */
#define	_XOPEN_LEGACY			-1
#define	_XOPEN_REALTIME			-1
#define	_XOPEN_REALTIME_THREADS		-1
#define	_XOPEN_UNIX			-1
#endif
# 150 "/usr/include/unistd.h" 3 4

/* Define the POSIX.2 version we target for compliance. */
#define	_POSIX2_VERSION		199212L

/*
 * POSIX-style system configuration variable accessors (for the
 * sysconf function).  The kernel does not directly implement the
 * sysconf() interface; rather, a C library stub translates references
 * to sysconf() into calls to sysctl() using a giant switch statement.
 * Those that are marked `user' are implemented entirely in the C
 * library and never query the kernel.  pathconf() is implemented
 * directly by the kernel so those are not defined here.
 */
#define	_SC_ARG_MAX		 1
#define	_SC_CHILD_MAX		 2
#define	_SC_CLK_TCK		 3
#define	_SC_NGROUPS_MAX		 4
#define	_SC_OPEN_MAX		 5
#define	_SC_JOB_CONTROL		 6
#define	_SC_SAVED_IDS		 7
#define	_SC_VERSION		 8
#define	_SC_BC_BASE_MAX		 9 /* user */
#define	_SC_BC_DIM_MAX		10 /* user */
#define	_SC_BC_SCALE_MAX	11 /* user */
#define	_SC_BC_STRING_MAX	12 /* user */
#define	_SC_COLL_WEIGHTS_MAX	13 /* user */
#define	_SC_EXPR_NEST_MAX	14 /* user */
#define	_SC_LINE_MAX		15 /* user */
#define	_SC_RE_DUP_MAX		16 /* user */
#define	_SC_2_VERSION		17 /* user */
#define	_SC_2_C_BIND		18 /* user */
#define	_SC_2_C_DEV		19 /* user */
#define	_SC_2_CHAR_TERM		20 /* user */
#define	_SC_2_FORT_DEV		21 /* user */
#define	_SC_2_FORT_RUN		22 /* user */
#define	_SC_2_LOCALEDEF		23 /* user */
#define	_SC_2_SW_DEV		24 /* user */
#define	_SC_2_UPE		25 /* user */
#define	_SC_STREAM_MAX		26 /* user */
#define	_SC_TZNAME_MAX		27 /* user */

#if __POSIX_VISIBLE >= 199309
#define	_SC_ASYNCHRONOUS_IO	28
#define	_SC_MAPPED_FILES	29
#define	_SC_MEMLOCK		30
#define	_SC_MEMLOCK_RANGE	31
#define	_SC_MEMORY_PROTECTION	32
#define	_SC_MESSAGE_PASSING	33
#define	_SC_PRIORITIZED_IO	34
#define	_SC_PRIORITY_SCHEDULING	35
#define	_SC_REALTIME_SIGNALS	36
#define	_SC_SEMAPHORES		37
#define	_SC_FSYNC		38
#define	_SC_SHARED_MEMORY_OBJECTS 39
#define	_SC_SYNCHRONIZED_IO	40
#define	_SC_TIMERS		41
#define	_SC_AIO_LISTIO_MAX	42
#define	_SC_AIO_MAX		43
#define	_SC_AIO_PRIO_DELTA_MAX	44
#define	_SC_DELAYTIMER_MAX	45
#define	_SC_MQ_OPEN_MAX		46
#define	_SC_PAGESIZE		47
#define	_SC_RTSIG_MAX		48
#define	_SC_SEM_NSEMS_MAX	49
#define	_SC_SEM_VALUE_MAX	50
#define	_SC_SIGQUEUE_MAX	51
#define	_SC_TIMER_MAX		52
#endif
# 218 "/usr/include/unistd.h" 3 4

#if __POSIX_VISIBLE >= 200112
#define	_SC_2_PBS		59 /* user */
#define	_SC_2_PBS_ACCOUNTING	60 /* user */
#define	_SC_2_PBS_CHECKPOINT	61 /* user */
#define	_SC_2_PBS_LOCATE	62 /* user */
#define	_SC_2_PBS_MESSAGE	63 /* user */
#define	_SC_2_PBS_TRACK		64 /* user */
#define	_SC_ADVISORY_INFO	65
#define	_SC_BARRIERS		66 /* user */
#define	_SC_CLOCK_SELECTION	67
#define	_SC_CPUTIME		68
#define	_SC_FILE_LOCKING	69
#define	_SC_GETGR_R_SIZE_MAX	70 /* user */
#define	_SC_GETPW_R_SIZE_MAX	71 /* user */
#define	_SC_HOST_NAME_MAX	72
#define	_SC_LOGIN_NAME_MAX	73
#define	_SC_MONOTONIC_CLOCK	74
#define	_SC_MQ_PRIO_MAX		75
#define	_SC_READER_WRITER_LOCKS	76 /* user */
#define	_SC_REGEXP		77 /* user */
#define	_SC_SHELL		78 /* user */
#define	_SC_SPAWN		79 /* user */
#define	_SC_SPIN_LOCKS		80 /* user */
#define	_SC_SPORADIC_SERVER	81
#define	_SC_THREAD_ATTR_STACKADDR 82 /* user */
#define	_SC_THREAD_ATTR_STACKSIZE 83 /* user */
#define	_SC_THREAD_CPUTIME	84 /* user */
#define	_SC_THREAD_DESTRUCTOR_ITERATIONS 85 /* user */
#define	_SC_THREAD_KEYS_MAX	86 /* user */
#define	_SC_THREAD_PRIO_INHERIT	87 /* user */
#define	_SC_THREAD_PRIO_PROTECT	88 /* user */
#define	_SC_THREAD_PRIORITY_SCHEDULING 89 /* user */
#define	_SC_THREAD_PROCESS_SHARED 90 /* user */
#define	_SC_THREAD_SAFE_FUNCTIONS 91 /* user */
#define	_SC_THREAD_SPORADIC_SERVER 92 /* user */
#define	_SC_THREAD_STACK_MIN	93 /* user */
#define	_SC_THREAD_THREADS_MAX	94 /* user */
#define	_SC_TIMEOUTS		95 /* user */
#define	_SC_THREADS		96 /* user */
#define	_SC_TRACE		97 /* user */
#define	_SC_TRACE_EVENT_FILTER	98 /* user */
#define	_SC_TRACE_INHERIT	99 /* user */
#define	_SC_TRACE_LOG		100 /* user */
#define	_SC_TTY_NAME_MAX	101 /* user */
#define	_SC_TYPED_MEMORY_OBJECTS 102
#define	_SC_V6_ILP32_OFF32	103 /* user */
#define	_SC_V6_ILP32_OFFBIG	104 /* user */
#define	_SC_V6_LP64_OFF64	105 /* user */
#define	_SC_V6_LPBIG_OFFBIG	106 /* user */
#define	_SC_IPV6		118
#define	_SC_RAW_SOCKETS		119
#define	_SC_SYMLOOP_MAX		120
#endif
# 272 "/usr/include/unistd.h" 3 4

#if __XSI_VISIBLE
#define	_SC_ATEXIT_MAX		107 /* user */
#define	_SC_IOV_MAX		56
#define	_SC_PAGE_SIZE		_SC_PAGESIZE
#define	_SC_XOPEN_CRYPT		108 /* user */
#define	_SC_XOPEN_ENH_I18N	109 /* user */
#define	_SC_XOPEN_LEGACY	110 /* user */
#define	_SC_XOPEN_REALTIME	111
#define	_SC_XOPEN_REALTIME_THREADS 112
#define	_SC_XOPEN_SHM		113
#define	_SC_XOPEN_STREAMS	114
#define	_SC_XOPEN_UNIX		115
#define	_SC_XOPEN_VERSION	116
#define	_SC_XOPEN_XCU_VERSION	117 /* user */
#endif
# 288 "/usr/include/unistd.h" 3 4

#if __BSD_VISIBLE
#define	_SC_NPROCESSORS_CONF	57
#define	_SC_NPROCESSORS_ONLN	58
#define	_SC_CPUSET_SIZE		122
#endif
# 294 "/usr/include/unistd.h" 3 4

/* Extensions found in Solaris and Linux. */
#define	_SC_PHYS_PAGES		121

/* Keys for the confstr(3) function. */
#if __POSIX_VISIBLE >= 199209
#define	_CS_PATH		1	/* default value of PATH */
#endif
# 302 "/usr/include/unistd.h" 3 4

#if __POSIX_VISIBLE >= 200112
#define	_CS_POSIX_V6_ILP32_OFF32_CFLAGS		2
#define	_CS_POSIX_V6_ILP32_OFF32_LDFLAGS	3
#define	_CS_POSIX_V6_ILP32_OFF32_LIBS		4
#define	_CS_POSIX_V6_ILP32_OFFBIG_CFLAGS	5
#define	_CS_POSIX_V6_ILP32_OFFBIG_LDFLAGS	6
#define	_CS_POSIX_V6_ILP32_OFFBIG_LIBS		7
#define	_CS_POSIX_V6_LP64_OFF64_CFLAGS		8
#define	_CS_POSIX_V6_LP64_OFF64_LDFLAGS		9
#define	_CS_POSIX_V6_LP64_OFF64_LIBS		10
#define	_CS_POSIX_V6_LPBIG_OFFBIG_CFLAGS	11
#define	_CS_POSIX_V6_LPBIG_OFFBIG_LDFLAGS	12
#define	_CS_POSIX_V6_LPBIG_OFFBIG_LIBS		13
#define	_CS_POSIX_V6_WIDTH_RESTRICTED_ENVS	14
#endif
# 318 "/usr/include/unistd.h" 3 4

__BEGIN_DECLS
/* 1003.1-1990 */
void	 _exit(int) __dead2;
int	 access(const char *, int);
unsigned int	 alarm(unsigned int);
int	 chdir(const char *);
int	 chown(const char *, uid_t, gid_t);
int	 close(int);
void	 closefrom(int);
int	 dup(int);
int	 dup2(int, int);
int	 execl(const char *, const char *, ...) __null_sentinel;
int	 execle(const char *, const char *, ...);
int	 execlp(const char *, const char *, ...) __null_sentinel;
int	 execv(const char *, char * const *);
int	 execve(const char *, char * const *, char * const *);
int	 execvp(const char *, char * const *);
pid_t	 fork(void);
long	 fpathconf(int, int);
char	*getcwd(char *, size_t);
gid_t	 getegid(void);
uid_t	 geteuid(void);
gid_t	 getgid(void);
int	 getgroups(int, gid_t []);
char	*getlogin(void);
pid_t	 getpgrp(void);
pid_t	 getpid(void);
pid_t	 getppid(void);
uid_t	 getuid(void);
int	 isatty(int);
int	 link(const char *, const char *);
#ifndef _LSEEK_DECLARED
#define	_LSEEK_DECLARED
off_t	 lseek(int, off_t, int);
#endif
# 354 "/usr/include/unistd.h" 3 4
long	 pathconf(const char *, int);
int	 pause(void);
int	 pipe(int *);
ssize_t	 read(int, void *, size_t);
int	 rmdir(const char *);
int	 setgid(gid_t);
int	 setpgid(pid_t, pid_t);
pid_t	 setsid(void);
int	 setuid(uid_t);
unsigned int	 sleep(unsigned int);
long	 sysconf(int);
pid_t	 tcgetpgrp(int);
int	 tcsetpgrp(int, pid_t);
char	*ttyname(int);
int	ttyname_r(int, char *, size_t);
int	 unlink(const char *);
ssize_t	 write(int, const void *, size_t);

/* 1003.2-1992 */
#if __POSIX_VISIBLE >= 199209 || __XSI_VISIBLE
size_t	 confstr(int, char *, size_t);
#ifndef _GETOPT_DECLARED
#define	_GETOPT_DECLARED
int	 getopt(int, char * const [], const char *);

extern char *optarg;			/* getopt(3) external variables */
extern int optind, opterr, optopt;
#endif /* _GETOPT_DECLARED */
# 382 "/usr/include/unistd.h" 3 4
#endif
# 383 "/usr/include/unistd.h" 3 4

/* ISO/IEC 9945-1: 1996 */
#if __POSIX_VISIBLE >= 199506 || __XSI_VISIBLE
int	 fsync(int);

/*
 * ftruncate() was in the POSIX Realtime Extension (it's used for shared
 * memory), but truncate() was not.
 */
#ifndef _FTRUNCATE_DECLARED
#define	_FTRUNCATE_DECLARED
int	 ftruncate(int, off_t);
#endif
# 396 "/usr/include/unistd.h" 3 4
#endif
# 397 "/usr/include/unistd.h" 3 4

#if __POSIX_VISIBLE >= 199506
int	 getlogin_r(char *, int);
#endif
# 401 "/usr/include/unistd.h" 3 4

/* 1003.1-2001 */
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE
int	 fchown(int, uid_t, gid_t);
ssize_t	 readlink(const char * __restrict, char * __restrict, size_t);
#endif
# 407 "/usr/include/unistd.h" 3 4
#if __POSIX_VISIBLE >= 200112
int	 gethostname(char *, size_t);
int	 setegid(gid_t);
int	 seteuid(uid_t);
#endif
# 412 "/usr/include/unistd.h" 3 4

/* 1003.1-2008 */
#if __POSIX_VISIBLE >= 200809 || __XSI_VISIBLE
int	 getsid(pid_t _pid);
int	 fchdir(int);
int	 getpgid(pid_t _pid);
int	 lchown(const char *, uid_t, gid_t);
ssize_t	 pread(int, void *, size_t, off_t);
ssize_t	 pwrite(int, const void *, size_t, off_t);

/* See comment at ftruncate() above. */
#ifndef _TRUNCATE_DECLARED
#define	_TRUNCATE_DECLARED
int	 truncate(const char *, off_t);
#endif
# 427 "/usr/include/unistd.h" 3 4
#endif /* __POSIX_VISIBLE >= 200809 || __XSI_VISIBLE */
# 428 "/usr/include/unistd.h" 3 4

#if __POSIX_VISIBLE >= 200809
int	faccessat(int, const char *, int, int);
int	fchownat(int, const char *, uid_t, gid_t, int);
int	fexecve(int, char *const [], char *const []);
int	linkat(int, const char *, int, const char *, int);
ssize_t	readlinkat(int, const char * __restrict, char * __restrict, size_t);
int	symlinkat(const char *, int, const char *);
int	unlinkat(int, const char *, int);
#endif /* __POSIX_VISIBLE >= 200809 */
# 438 "/usr/include/unistd.h" 3 4

/*
 * symlink() was originally in POSIX.1a, which was withdrawn after
 * being overtaken by events (1003.1-2001).  It was in XPG4.2, and of
 * course has been in BSD since 4.2.
 */
#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE >= 402
int	 symlink(const char * __restrict, const char * __restrict);
#endif
# 447 "/usr/include/unistd.h" 3 4

/* X/Open System Interfaces */
#if __XSI_VISIBLE
char	*crypt(const char *, const char *);
/* char	*ctermid(char *); */		/* XXX ??? */
int	 encrypt(char *, int);
long	 gethostid(void);
int	 lockf(int, int, off_t);
int	 nice(int);
int	 setregid(gid_t, gid_t);
int	 setreuid(uid_t, uid_t);

#ifndef _SWAB_DECLARED
#define _SWAB_DECLARED
void	 swab(const void * __restrict, void * __restrict, ssize_t);
#endif /* _SWAB_DECLARED */
# 463 "/usr/include/unistd.h" 3 4

void	 sync(void);

#endif /* __XSI_VISIBLE */
# 467 "/usr/include/unistd.h" 3 4

#if (__XSI_VISIBLE && __XSI_VISIBLE <= 500) || __BSD_VISIBLE
int	 brk(const void *);
int	 chroot(const char *);
int	 getdtablesize(void);
int	 getpagesize(void) __pure2;
char	*getpass(const char *);
void	*sbrk(intptr_t);
#endif
# 476 "/usr/include/unistd.h" 3 4

#if (__XSI_VISIBLE && __XSI_VISIBLE <= 600) || __BSD_VISIBLE
char	*getwd(char *);			/* obsoleted by getcwd() */
useconds_t
	 ualarm(useconds_t, useconds_t);
int	 usleep(useconds_t);
pid_t	 vfork(void) __returns_twice;
#endif
# 484 "/usr/include/unistd.h" 3 4

#if __BSD_VISIBLE
struct timeval;				/* select(2) */
int	 acct(const char *);
int	 async_daemon(void);
int	 check_utility_compat(const char *);
const char *
	 crypt_get_format(void);
int	 crypt_set_format(const char *);
int	 des_cipher(const char *, char *, long, int);
int	 des_setkey(const char *key);
int	 dup3(int, int, int);
int	 eaccess(const char *, int);
void	 endusershell(void);
int	 exect(const char *, char * const *, char * const *);
int	 execvP(const char *, const char *, char * const *);
int	 feature_present(const char *);
char	*fflagstostr(u_long);
int	 getdomainname(char *, int);
int	 getgrouplist(const char *, gid_t, gid_t *, int *);
int	 getloginclass(char *, size_t);
mode_t	 getmode(const void *, mode_t);
int	 getosreldate(void);
int	 getpeereid(int, uid_t *, gid_t *);
int	 getresgid(gid_t *, gid_t *, gid_t *);
int	 getresuid(uid_t *, uid_t *, uid_t *);
char	*getusershell(void);
int	 initgroups(const char *, gid_t);
int	 iruserok(unsigned long, int, const char *, const char *);
int	 iruserok_sa(const void *, int, int, const char *, const char *);
int	 issetugid(void);
void	__FreeBSD_libc_enter_restricted_mode(void);
long	 lpathconf(const char *, int);
#ifndef _MKDTEMP_DECLARED
char	*mkdtemp(char *);
#define	_MKDTEMP_DECLARED
#endif
# 521 "/usr/include/unistd.h" 3 4
#ifndef	_MKNOD_DECLARED
int	 mknod(const char *, mode_t, dev_t);
#define	_MKNOD_DECLARED
#endif
# 525 "/usr/include/unistd.h" 3 4
#ifndef _MKSTEMP_DECLARED
int	 mkstemp(char *);
#define	_MKSTEMP_DECLARED
#endif
# 529 "/usr/include/unistd.h" 3 4
int	 mkstemps(char *, int);
#ifndef _MKTEMP_DECLARED
char	*mktemp(char *);
#define	_MKTEMP_DECLARED
#endif
# 534 "/usr/include/unistd.h" 3 4
int	 nfssvc(int, void *);
int	 nlm_syscall(int, int, int, char **);
int	 pipe2(int *, int);
int	 profil(char *, size_t, vm_offset_t, int);
int	 rcmd(char **, int, const char *, const char *, const char *, int *);
int	 rcmd_af(char **, int, const char *,
		const char *, const char *, int *, int);
int	 rcmdsh(char **, int, const char *,
		const char *, const char *, const char *);
char	*re_comp(const char *);
int	 re_exec(const char *);
int	 reboot(int);
int	 revoke(const char *);
pid_t	 rfork(int);
pid_t	 rfork_thread(int, void *, int (*)(void *), void *);
int	 rresvport(int *);
int	 rresvport_af(int *, int);
int	 ruserok(const char *, int, const char *, const char *);
#if __BSD_VISIBLE
#ifndef _SELECT_DECLARED
#define	_SELECT_DECLARED
int	 select(int, fd_set *, fd_set *, fd_set *, struct timeval *);
#endif
# 557 "/usr/include/unistd.h" 3 4
#endif
# 558 "/usr/include/unistd.h" 3 4
int	 setdomainname(const char *, int);
int	 setgroups(int, const gid_t *);
void	 sethostid(long);
int	 sethostname(const char *, int);
#ifndef _SETKEY_DECLARED
int	 setkey(const char *);
#define	_SETKEY_DECLARED
#endif
# 566 "/usr/include/unistd.h" 3 4
int	 setlogin(const char *);
int	 setloginclass(const char *);
void	*setmode(const char *);
int	 setpgrp(pid_t, pid_t);			/* obsoleted by setpgid() */
void	 setproctitle(const char *_fmt, ...) __printf0like(1, 2);
int	 setresgid(gid_t, gid_t, gid_t);
int	 setresuid(uid_t, uid_t, uid_t);
int	 setrgid(gid_t);
int	 setruid(uid_t);
void	 setusershell(void);
int	 strtofflags(char **, u_long *, u_long *);
int	 swapon(const char *);
int	 swapoff(const char *);
int	 syscall(int, ...);
off_t	 __syscall(quad_t, ...);
int	 undelete(const char *);
int	 unwhiteout(const char *);
void	*valloc(size_t);			/* obsoleted by malloc() */

#ifndef _OPTRESET_DECLARED
#define	_OPTRESET_DECLARED
extern int optreset;			/* getopt(3) external variable */
#endif
# 589 "/usr/include/unistd.h" 3 4
#endif /* __BSD_VISIBLE */
# 590 "/usr/include/unistd.h" 3 4
__END_DECLS

#endif /* !_UNISTD_H_ */
# 593 "/usr/include/unistd.h" 3 4
# 68 "sys/unix/erl_unix_sys.h" 2
#endif
# 69 "sys/unix/erl_unix_sys.h"

#ifndef HAVE_MMAP
#   define HAVE_MMAP 0
#endif
# 73 "sys/unix/erl_unix_sys.h"

#if HAVE_MMAP
#if 0 /* expanded by -frewrite-includes */
#   include <sys/mman.h>
#endif /* expanded by -frewrite-includes */
# 75 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/sys/mman.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)mman.h	8.2 (Berkeley) 1/9/95
 * $FreeBSD: head/sys/sys/mman.h 296162 2016-02-28 17:52:33Z kib $
 */

#ifndef _SYS_MMAN_H_
#define _SYS_MMAN_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/mman.h" 3 4
# 37 "/usr/include/sys/mman.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/sys/mman.h" 3 4
# 38 "/usr/include/sys/mman.h" 3 4

#if __BSD_VISIBLE
/*
 * Inheritance for minherit()
 */
#define INHERIT_SHARE	0
#define INHERIT_COPY	1
#define INHERIT_NONE	2
#endif
# 47 "/usr/include/sys/mman.h" 3 4

/*
 * Protections are chosen from these bits, or-ed together
 */
#define	PROT_NONE	0x00	/* no permissions */
#define	PROT_READ	0x01	/* pages can be read */
#define	PROT_WRITE	0x02	/* pages can be written */
#define	PROT_EXEC	0x04	/* pages can be executed */

/*
 * Flags contain sharing type and options.
 * Sharing types; choose one.
 */
#define	MAP_SHARED	0x0001		/* share changes */
#define	MAP_PRIVATE	0x0002		/* changes are private */
#if __BSD_VISIBLE
#define	MAP_COPY	MAP_PRIVATE	/* Obsolete */
#endif
# 65 "/usr/include/sys/mman.h" 3 4

/*
 * Other flags
 */
#define	MAP_FIXED	 0x0010	/* map addr must be exactly as requested */

#if __BSD_VISIBLE
#define	MAP_RESERVED0020 0x0020	/* previously unimplemented MAP_RENAME */
#define	MAP_RESERVED0040 0x0040	/* previously unimplemented MAP_NORESERVE */
#define	MAP_RESERVED0080 0x0080	/* previously misimplemented MAP_INHERIT */
#define	MAP_RESERVED0100 0x0100	/* previously unimplemented MAP_NOEXTEND */
#define	MAP_HASSEMAPHORE 0x0200	/* region may contain semaphores */
#define	MAP_STACK	 0x0400	/* region grows down, like a stack */
#define	MAP_NOSYNC	 0x0800 /* page to but do not sync underlying file */

/*
 * Mapping type
 */
#define	MAP_FILE	 0x0000	/* map from file (default) */
#define	MAP_ANON	 0x1000	/* allocated from memory, swap space */
#ifndef _KERNEL
#define	MAP_ANONYMOUS	 MAP_ANON /* For compatibility. */
#endif /* !_KERNEL */
# 88 "/usr/include/sys/mman.h" 3 4

/*
 * Extended flags
 */
#define	MAP_EXCL	 0x00004000 /* for MAP_FIXED, fail if address is used */
#define	MAP_NOCORE	 0x00020000 /* dont include these pages in a coredump */
#define	MAP_PREFAULT_READ 0x00040000 /* prefault mapping for reading */
#ifdef __LP64__
#define	MAP_32BIT	 0x00080000 /* map in the low 2GB of address space */
#endif
# 98 "/usr/include/sys/mman.h" 3 4

/*
 * Request specific alignment (n == log2 of the desired alignment).
 *
 * MAP_ALIGNED_SUPER requests optimal superpage alignment, but does
 * not enforce a specific alignment.
 */
#define	MAP_ALIGNED(n)	 ((n) << MAP_ALIGNMENT_SHIFT)
#define	MAP_ALIGNMENT_SHIFT	24
#define	MAP_ALIGNMENT_MASK	MAP_ALIGNED(0xff)
#define	MAP_ALIGNED_SUPER	MAP_ALIGNED(1) /* align on a superpage */
#endif /* __BSD_VISIBLE */
# 110 "/usr/include/sys/mman.h" 3 4

#if __POSIX_VISIBLE >= 199309
/*
 * Process memory locking
 */
#define MCL_CURRENT	0x0001	/* Lock only current memory */
#define MCL_FUTURE	0x0002	/* Lock all future memory as well */
#endif
# 118 "/usr/include/sys/mman.h" 3 4

/*
 * Error return from mmap()
 */
#define MAP_FAILED	((void *)-1)

/*
 * msync() flags
 */
#define	MS_SYNC		0x0000	/* msync synchronously */
#define MS_ASYNC	0x0001	/* return immediately */
#define MS_INVALIDATE	0x0002	/* invalidate all cached data */

/*
 * Advice to madvise
 */
#define	_MADV_NORMAL	0	/* no further special treatment */
#define	_MADV_RANDOM	1	/* expect random page references */
#define	_MADV_SEQUENTIAL 2	/* expect sequential page references */
#define	_MADV_WILLNEED	3	/* will need these pages */
#define	_MADV_DONTNEED	4	/* dont need these pages */

#if __BSD_VISIBLE
#define	MADV_NORMAL	_MADV_NORMAL
#define	MADV_RANDOM	_MADV_RANDOM
#define	MADV_SEQUENTIAL _MADV_SEQUENTIAL
#define	MADV_WILLNEED	_MADV_WILLNEED
#define	MADV_DONTNEED	_MADV_DONTNEED
#define	MADV_FREE	5	/* dont need these pages, and junk contents */
#define	MADV_NOSYNC	6	/* try to avoid flushes to physical media */
#define	MADV_AUTOSYNC	7	/* revert to default flushing strategy */
#define	MADV_NOCORE	8	/* do not include these pages in a core file */
#define	MADV_CORE	9	/* revert to including pages in a core file */
#define	MADV_PROTECT	10	/* protect process from pageout kill */

/*
 * Return bits from mincore
 */
#define	MINCORE_INCORE	 	 0x1 /* Page is incore */
#define	MINCORE_REFERENCED	 0x2 /* Page has been referenced by us */
#define	MINCORE_MODIFIED	 0x4 /* Page has been modified by us */
#define	MINCORE_REFERENCED_OTHER 0x8 /* Page has been referenced */
#define	MINCORE_MODIFIED_OTHER	0x10 /* Page has been modified */
#define	MINCORE_SUPER		0x20 /* Page is a "super" page */

/*
 * Anonymous object constant for shm_open().
 */
#define	SHM_ANON		((char *)1)
#endif /* __BSD_VISIBLE */
# 168 "/usr/include/sys/mman.h" 3 4

/*
 * XXX missing POSIX_TYPED_MEM_* macros and
 * posix_typed_mem_info structure.
 */
#if __POSIX_VISIBLE >= 200112
#define	POSIX_MADV_NORMAL	_MADV_NORMAL
#define	POSIX_MADV_RANDOM	_MADV_RANDOM
#define	POSIX_MADV_SEQUENTIAL	_MADV_SEQUENTIAL
#define	POSIX_MADV_WILLNEED	_MADV_WILLNEED
#define	POSIX_MADV_DONTNEED	_MADV_DONTNEED
#endif
# 180 "/usr/include/sys/mman.h" 3 4

#ifndef _MODE_T_DECLARED
typedef	__mode_t	mode_t;
#define	_MODE_T_DECLARED
#endif
# 185 "/usr/include/sys/mman.h" 3 4

#ifndef _OFF_T_DECLARED
typedef	__off_t		off_t;
#define	_OFF_T_DECLARED
#endif
# 190 "/usr/include/sys/mman.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 195 "/usr/include/sys/mman.h" 3 4

#if defined(_KERNEL) || defined(_WANT_FILE)
#if 0 /* expanded by -frewrite-includes */
#include <sys/lock.h>
#endif /* expanded by -frewrite-includes */
# 197 "/usr/include/sys/mman.h" 3 4
# 198 "/usr/include/sys/mman.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/mutex.h>
#endif /* expanded by -frewrite-includes */
# 198 "/usr/include/sys/mman.h" 3 4
# 199 "/usr/include/sys/mman.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/queue.h>
#endif /* expanded by -frewrite-includes */
# 199 "/usr/include/sys/mman.h" 3 4
# 200 "/usr/include/sys/mman.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/rangelock.h>
#endif /* expanded by -frewrite-includes */
# 200 "/usr/include/sys/mman.h" 3 4
# 201 "/usr/include/sys/mman.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <vm/vm.h>
#endif /* expanded by -frewrite-includes */
# 201 "/usr/include/sys/mman.h" 3 4
# 202 "/usr/include/sys/mman.h" 3 4

struct file;

struct shmfd {
	size_t		shm_size;
	vm_object_t	shm_object;
	int		shm_refs;
	uid_t		shm_uid;
	gid_t		shm_gid;
	mode_t		shm_mode;
	int		shm_kmappings;

	/*
	 * Values maintained solely to make this a better-behaved file
	 * descriptor for fstat() to run on.
	 */
	struct timespec	shm_atime;
	struct timespec	shm_mtime;
	struct timespec	shm_ctime;
	struct timespec	shm_birthtime;
	ino_t		shm_ino;

	struct label	*shm_label;		/* MAC label */
	const char	*shm_path;

	struct rangelock shm_rl;
	struct mtx	shm_mtx;
};
#endif
# 231 "/usr/include/sys/mman.h" 3 4

#ifdef _KERNEL
int	shm_map(struct file *fp, size_t size, off_t offset, void **memp);
int	shm_unmap(struct file *fp, void *mem, size_t size);

int	shm_access(struct shmfd *shmfd, struct ucred *ucred, int flags);
struct shmfd *shm_alloc(struct ucred *ucred, mode_t mode);
struct shmfd *shm_hold(struct shmfd *shmfd);
void	shm_drop(struct shmfd *shmfd);
int	shm_dotruncate(struct shmfd *shmfd, off_t length);

extern struct fileops shm_ops;
#else /* !_KERNEL */
# 244 "/usr/include/sys/mman.h" 3 4

__BEGIN_DECLS
/*
 * XXX not yet implemented: posix_mem_offset(), posix_typed_mem_get_info(),
 * posix_typed_mem_open().
 */
#if __BSD_VISIBLE
int	getpagesizes(size_t *, int);
int	madvise(void *, size_t, int);
int	mincore(const void *, size_t, char *);
int	minherit(void *, size_t, int);
#endif
# 256 "/usr/include/sys/mman.h" 3 4
int	mlock(const void *, size_t);
#ifndef _MMAP_DECLARED
#define	_MMAP_DECLARED
void *	mmap(void *, size_t, int, int, int, off_t);
#endif
# 261 "/usr/include/sys/mman.h" 3 4
int	mprotect(const void *, size_t, int);
int	msync(void *, size_t, int);
int	munlock(const void *, size_t);
int	munmap(void *, size_t);
#if __POSIX_VISIBLE >= 200112
int	posix_madvise(void *, size_t, int);
#endif
# 268 "/usr/include/sys/mman.h" 3 4
#if __POSIX_VISIBLE >= 199309
int	mlockall(int);
int	munlockall(void);
int	shm_open(const char *, int, mode_t);
int	shm_unlink(const char *);
#endif
# 274 "/usr/include/sys/mman.h" 3 4
__END_DECLS

#endif /* !_KERNEL */
# 277 "/usr/include/sys/mman.h" 3 4

#endif /* !_SYS_MMAN_H_ */
# 279 "/usr/include/sys/mman.h" 3 4
# 76 "sys/unix/erl_unix_sys.h" 2
#endif
# 77 "sys/unix/erl_unix_sys.h"

#if TIME_WITH_SYS_TIME
#if 0 /* expanded by -frewrite-includes */
#   include <sys/time.h>
#endif /* expanded by -frewrite-includes */
# 79 "sys/unix/erl_unix_sys.h"
# 80 "sys/unix/erl_unix_sys.h"
#if 0 /* expanded by -frewrite-includes */
#   include <time.h>
#endif /* expanded by -frewrite-includes */
# 80 "sys/unix/erl_unix_sys.h"
# 81 "sys/unix/erl_unix_sys.h"
#else
# 82 "sys/unix/erl_unix_sys.h"
#   if HAVE_SYS_TIME_H
#if 0 /* expanded by -frewrite-includes */
#       include <sys/time.h>
#endif /* expanded by -frewrite-includes */
# 83 "sys/unix/erl_unix_sys.h"
# 84 "sys/unix/erl_unix_sys.h"
#   else
# 85 "sys/unix/erl_unix_sys.h"
#if 0 /* expanded by -frewrite-includes */
#       include <time.h>
#endif /* expanded by -frewrite-includes */
# 85 "sys/unix/erl_unix_sys.h"
# 86 "sys/unix/erl_unix_sys.h"
#   endif
# 87 "sys/unix/erl_unix_sys.h"
#endif
# 88 "sys/unix/erl_unix_sys.h"

#if 0 /* expanded by -frewrite-includes */
#include <sys/times.h>
#endif /* expanded by -frewrite-includes */
# 89 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/sys/times.h" 1 3 4
/*-
 * Copyright (c) 1990, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)times.h	8.4 (Berkeley) 1/21/94
 * $FreeBSD: head/sys/sys/times.h 127976 2004-04-07 04:19:52Z imp $
 */

#ifndef	_SYS_TIMES_H_
#define	_SYS_TIMES_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/sys/times.h" 3 4
# 42 "/usr/include/sys/times.h" 3 4

#ifndef _CLOCK_T_DECLARED
typedef	__clock_t	clock_t;
#define	_CLOCK_T_DECLARED
#endif
# 47 "/usr/include/sys/times.h" 3 4

struct tms {
	clock_t tms_utime;	/* User CPU time */
	clock_t tms_stime;	/* System CPU time */
	clock_t tms_cutime;	/* User CPU time of terminated child procs */
	clock_t tms_cstime;	/* System CPU time of terminated child procs */
};

#ifndef _KERNEL
#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 56 "/usr/include/sys/times.h" 3 4
# 57 "/usr/include/sys/times.h" 3 4

__BEGIN_DECLS
clock_t	times(struct tms *);
__END_DECLS
#endif
# 62 "/usr/include/sys/times.h" 3 4
#endif /* !_SYS_TIMES_H_ */
# 63 "/usr/include/sys/times.h" 3 4
# 90 "sys/unix/erl_unix_sys.h" 2

#ifdef HAVE_IEEEFP_H
#if 0 /* expanded by -frewrite-includes */
#include <ieeefp.h>
#endif /* expanded by -frewrite-includes */
# 92 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/ieeefp.h" 1 3 4
/*	$NetBSD: ieeefp.h,v 1.4 1998/01/09 08:03:43 perry Exp $	*/
/* $FreeBSD: head/include/ieeefp.h 226607 2011-10-21 06:41:46Z das $ */

/* 
 * Written by J.T. Conklin, Apr 6, 1995
 * Public domain.
 */

#ifndef _IEEEFP_H_
#define _IEEEFP_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 12 "/usr/include/ieeefp.h" 3 4
# 13 "/usr/include/ieeefp.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/ieeefp.h>
#endif /* expanded by -frewrite-includes */
# 13 "/usr/include/ieeefp.h" 3 4
# 1 "/usr/include/machine/ieeefp.h" 1 3 4
/*	$NetBSD: ieeefp.h,v 1.1 2001/01/10 19:02:06 bjh21 Exp $	*/
/* $FreeBSD: head/sys/arm/include/ieeefp.h 226607 2011-10-21 06:41:46Z das $ */
/*-
 * Based on ieeefp.h written by J.T. Conklin, Apr 28, 1995
 * Public domain.
 */

#ifndef _MACHINE_IEEEFP_H_
#define _MACHINE_IEEEFP_H_

/* Deprecated historical FPU control interface */

/* FP exception codes */
#define FP_EXCEPT_INV	0
#define FP_EXCEPT_DZ	1
#define FP_EXCEPT_OFL	2
#define FP_EXCEPT_UFL	3
#define FP_EXCEPT_IMP	4

/* Exception type (used by fpsetmask() et al.) */

typedef int fp_except;

/* Bit defines for fp_except */

#define	FP_X_INV	(1 << FP_EXCEPT_INV)	/* invalid operation exception */
#define	FP_X_DZ		(1 << FP_EXCEPT_DZ)	/* divide-by-zero exception */
#define	FP_X_OFL	(1 << FP_EXCEPT_OFL)	/* overflow exception */
#define	FP_X_UFL	(1 << FP_EXCEPT_UFL)	/* underflow exception */
#define	FP_X_IMP	(1 << FP_EXCEPT_IMP)	/* imprecise (loss of precision; "inexact") */

/* Rounding modes */

typedef enum {
    FP_RN=0,			/* round to nearest representable number */
    FP_RP=1,			/* round toward positive infinity */
    FP_RM=2,			/* round toward negative infinity */
    FP_RZ=3			/* round to zero (truncate) */
} fp_rnd_t;

/*
 * FP precision modes
 */
typedef enum {
	FP_PS=0,	/* 24 bit (single-precision) */
	FP_PRS,		/* reserved */
	FP_PD,		/* 53 bit (double-precision) */
	FP_PE		/* 64 bit (extended-precision) */
} fp_prec_t;

#define fp_except_t	int

#endif /* _MACHINE_IEEEFP_H_ */
# 54 "/usr/include/machine/ieeefp.h" 3 4
# 14 "/usr/include/ieeefp.h" 2 3 4

#endif /* _IEEEFP_H_ */
# 16 "/usr/include/ieeefp.h" 3 4
# 93 "sys/unix/erl_unix_sys.h" 2
#endif
# 94 "sys/unix/erl_unix_sys.h"

#ifdef QNX
#if 0 /* expanded by -frewrite-includes */
#include <process.h>
#endif /* expanded by -frewrite-includes */
# 96 "sys/unix/erl_unix_sys.h"
# 97 "sys/unix/erl_unix_sys.h"
#if 0 /* expanded by -frewrite-includes */
#include <sys/qnx_glob.h>
#endif /* expanded by -frewrite-includes */
# 97 "sys/unix/erl_unix_sys.h"
# 98 "sys/unix/erl_unix_sys.h"
#endif
# 99 "sys/unix/erl_unix_sys.h"

#if 0 /* expanded by -frewrite-includes */
#include <pwd.h>
#endif /* expanded by -frewrite-includes */
# 100 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/pwd.h" 1 3 4
/*-
 * Copyright (c) 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 * (c) UNIX System Laboratories, Inc.
 * All or some portions of this file are derived from material licensed
 * to the University of California by American Telephone and Telegraph
 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
 * the permission of UNIX System Laboratories, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)pwd.h	8.2 (Berkeley) 1/21/94
 * $FreeBSD: head/include/pwd.h 241731 2012-10-19 12:44:22Z brooks $
 */

#ifndef _PWD_H_
#define	_PWD_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/pwd.h" 3 4
# 42 "/usr/include/pwd.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 42 "/usr/include/pwd.h" 3 4
# 43 "/usr/include/pwd.h" 3 4

#ifndef _GID_T_DECLARED
typedef	__gid_t		gid_t;
#define	_GID_T_DECLARED
#endif
# 48 "/usr/include/pwd.h" 3 4

#ifndef _TIME_T_DECLARED
typedef	__time_t	time_t;
#define	_TIME_T_DECLARED
#endif
# 53 "/usr/include/pwd.h" 3 4

#ifndef _UID_T_DECLARED
typedef	__uid_t		uid_t;
#define	_UID_T_DECLARED
#endif
# 58 "/usr/include/pwd.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef __size_t	size_t;
#define _SIZE_T_DECLARED
#endif
# 63 "/usr/include/pwd.h" 3 4

#define _PATH_PWD		"/etc"
#define	_PATH_PASSWD		"/etc/passwd"
#define	_PASSWD			"passwd"
#define	_PATH_MASTERPASSWD	"/etc/master.passwd"
#define	_MASTERPASSWD		"master.passwd"

#define	_PATH_MP_DB		"/etc/pwd.db"
#define	_MP_DB			"pwd.db"
#define	_PATH_SMP_DB		"/etc/spwd.db"
#define	_SMP_DB			"spwd.db"

#define	_PATH_PWD_MKDB		"/usr/sbin/pwd_mkdb"

/* Historically, the keys in _PATH_MP_DB/_PATH_SMP_DB had the format
 * `1 octet tag | key', where the tag is one of the _PW_KEY* values
 * listed below.  These values happen to be ASCII digits.  Starting
 * with FreeBSD 5.1, the tag is now still a single octet, but the
 * upper 4 bits are interpreted as a version.  Pre-FreeBSD 5.1 format
 * entries are version `3' -- this conveniently results in the same
 * key values as before.  The new, architecture-independent entries
 * are version `4'.
 * As it happens, some applications read the database directly.
 * (Bad app, no cookie!)  Thus, we leave the _PW_KEY* symbols at their
 * old pre-FreeBSD 5.1 values so these apps still work.  Consequently
 * we have to muck around a bit more to get the correct, versioned
 * tag, and that is what the _PW_VERSIONED macro is about.
 */

#define _PW_VERSION_MASK	'\xF0'
#define _PW_VERSIONED(x, v)	((unsigned char)(((x) & 0xCF) | ((v)<<4)))

#define	_PW_KEYBYNAME		'\x31'	/* stored by name */
#define	_PW_KEYBYNUM		'\x32'	/* stored by entry in the "file" */
#define	_PW_KEYBYUID		'\x33'	/* stored by uid */
#define _PW_KEYYPENABLED	'\x34'	/* YP is enabled */
#define	_PW_KEYYPBYNUM		'\x35'	/* special +@netgroup entries */

/* The database also contains a key to indicate the format version of
 * the entries therein.  There may be other, older versioned entries
 * as well.
 */
#define _PWD_VERSION_KEY	"\xFF" "VERSION"
#define _PWD_CURRENT_VERSION	'\x04'

#define	_PASSWORD_EFMT1		'_'	/* extended encryption format */

#define	_PASSWORD_LEN		128	/* max length, not counting NULL */

struct passwd {
	char	*pw_name;		/* user name */
	char	*pw_passwd;		/* encrypted password */
	uid_t	pw_uid;			/* user uid */
	gid_t	pw_gid;			/* user gid */
	time_t	pw_change;		/* password change time */
	char	*pw_class;		/* user access class */
	char	*pw_gecos;		/* Honeywell login info */
	char	*pw_dir;		/* home directory */
	char	*pw_shell;		/* default shell */
	time_t	pw_expire;		/* account expiration */
	int	pw_fields;		/* internal: fields filled in */
};

/* Mapping from fields to bits for pw_fields. */
#define _PWF(x)		(1 << x)
#define _PWF_NAME	_PWF(0)
#define _PWF_PASSWD	_PWF(1)
#define _PWF_UID	_PWF(2)
#define _PWF_GID	_PWF(3)
#define _PWF_CHANGE	_PWF(4)
#define _PWF_CLASS	_PWF(5)
#define _PWF_GECOS	_PWF(6)
#define _PWF_DIR	_PWF(7)
#define _PWF_SHELL	_PWF(8)
#define _PWF_EXPIRE	_PWF(9)

/* XXX These flags are bogus.  With nsswitch, there are many
 * possible sources and they cannot be represented in a small integer.
 */                           
#define _PWF_SOURCE	0x3000
#define _PWF_FILES	0x1000
#define _PWF_NIS	0x2000
#define _PWF_HESIOD	0x3000

__BEGIN_DECLS
struct passwd	*getpwnam(const char *);
struct passwd	*getpwuid(uid_t);

#if __XSI_VISIBLE >= 500
void		 endpwent(void);
struct passwd	*getpwent(void);
void		 setpwent(void);
#endif
# 156 "/usr/include/pwd.h" 3 4

#if __POSIX_VISIBLE >= 200112 || __XSI_VISIBLE >= 500
int		 getpwnam_r(const char *, struct passwd *, char *, size_t,
		    struct passwd **);
int		 getpwuid_r(uid_t, struct passwd *, char *, size_t,
		    struct passwd **);
#endif
# 163 "/usr/include/pwd.h" 3 4

#if __BSD_VISIBLE
int		 getpwent_r(struct passwd *, char *, size_t, struct passwd **);
int		 setpassent(int);
const char	*user_from_uid(uid_t, int);
int		 uid_from_user(const char *, uid_t *);
int		 pwcache_userdb(int (*)(int), void (*)(void),
		    struct passwd * (*)(const char *),
		    struct passwd * (*)(uid_t));
#endif
# 173 "/usr/include/pwd.h" 3 4
__END_DECLS

#endif /* !_PWD_H_ */
# 176 "/usr/include/pwd.h" 3 4
# 101 "sys/unix/erl_unix_sys.h" 2

#ifndef HZ
#define HZ 60
#endif
# 105 "sys/unix/erl_unix_sys.h"

#ifdef NETDB_H_NEEDS_IN_H
#if 0 /* expanded by -frewrite-includes */
#include <netinet/in.h>
#endif /* expanded by -frewrite-includes */
# 107 "sys/unix/erl_unix_sys.h"
# 108 "sys/unix/erl_unix_sys.h"
#endif
# 109 "sys/unix/erl_unix_sys.h"
#if 0 /* expanded by -frewrite-includes */
#include <netdb.h>
#endif /* expanded by -frewrite-includes */
# 109 "sys/unix/erl_unix_sys.h"
# 1 "/usr/include/netdb.h" 1 3 4
/*-
 * Copyright (c) 1980, 1983, 1988, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * -
 * Portions Copyright (c) 1993 by Digital Equipment Corporation.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies, and that
 * the name of Digital Equipment Corporation not be used in advertising or
 * publicity pertaining to distribution of the document or software without
 * specific, written prior permission.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND DIGITAL EQUIPMENT CORP. DISCLAIMS ALL
 * WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS.   IN NO EVENT SHALL DIGITAL EQUIPMENT
 * CORPORATION BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
 * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
 * SOFTWARE.
 * -
 * --Copyright--
 */

/*
 *      @(#)netdb.h	8.1 (Berkeley) 6/2/93
 *      From: Id: netdb.h,v 8.9 1996/11/19 08:39:29 vixie Exp $
 * $FreeBSD: head/include/netdb.h 292514 2015-12-20 15:18:50Z ume $
 */

#ifndef _NETDB_H_
#define _NETDB_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 60 "/usr/include/netdb.h" 3 4
# 61 "/usr/include/netdb.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 61 "/usr/include/netdb.h" 3 4
# 62 "/usr/include/netdb.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 67 "/usr/include/netdb.h" 3 4

#ifndef _SOCKLEN_T_DECLARED
typedef	__socklen_t	socklen_t;
#define	_SOCKLEN_T_DECLARED
#endif
# 72 "/usr/include/netdb.h" 3 4

#ifndef _UINT32_T_DECLARED
typedef	__uint32_t	uint32_t;
#define	_UINT32_T_DECLARED
#endif
# 77 "/usr/include/netdb.h" 3 4

#ifndef _PATH_HEQUIV
# define	_PATH_HEQUIV	"/etc/hosts.equiv"
#endif
# 81 "/usr/include/netdb.h" 3 4
#define	_PATH_HOSTS	"/etc/hosts"
#define	_PATH_NETWORKS	"/etc/networks"
#define	_PATH_PROTOCOLS	"/etc/protocols"
#define	_PATH_SERVICES	"/etc/services"
#define	_PATH_SERVICES_DB "/var/db/services.db"

#define	h_errno (*__h_errno())

/*
 * Structures returned by network data base library.  All addresses are
 * supplied in host order, and returned in network order (suitable for
 * use in system calls).
 */
struct hostent {
	char	*h_name;	/* official name of host */
	char	**h_aliases;	/* alias list */
	int	h_addrtype;	/* host address type */
	int	h_length;	/* length of address */
	char	**h_addr_list;	/* list of addresses from name server */
#define	h_addr	h_addr_list[0]	/* address, for backward compatibility */
};

struct netent {
	char		*n_name;	/* official name of net */
	char		**n_aliases;	/* alias list */
	int		n_addrtype;	/* net address type */
	uint32_t	n_net;		/* network # */
};

struct servent {
	char	*s_name;	/* official service name */
	char	**s_aliases;	/* alias list */
	int	s_port;		/* port # */
	char	*s_proto;	/* protocol to use */
};

struct protoent {
	char	*p_name;	/* official protocol name */
	char	**p_aliases;	/* alias list */
	int	p_proto;	/* protocol # */
};

struct addrinfo {
	int	ai_flags;	/* AI_PASSIVE, AI_CANONNAME, AI_NUMERICHOST */
	int	ai_family;	/* AF_xxx */
	int	ai_socktype;	/* SOCK_xxx */
	int	ai_protocol;	/* 0 or IPPROTO_xxx for IPv4 and IPv6 */
	socklen_t ai_addrlen;	/* length of ai_addr */
	char	*ai_canonname;	/* canonical name for hostname */
	struct	sockaddr *ai_addr;	/* binary address */
	struct	addrinfo *ai_next;	/* next structure in linked list */
};

/*
 * Error return codes from gethostbyname() and gethostbyaddr()
 * (left in h_errno).
 */

#define	NETDB_INTERNAL	-1	/* see errno */
#define	NETDB_SUCCESS	0	/* no problem */
#define	HOST_NOT_FOUND	1 /* Authoritative Answer Host not found */
#define	TRY_AGAIN	2 /* Non-Authoritative Host not found, or SERVERFAIL */
#define	NO_RECOVERY	3 /* Non recoverable errors, FORMERR, REFUSED, NOTIMP */
#define	NO_DATA		4 /* Valid name, no data record of requested type */
#define	NO_ADDRESS	NO_DATA		/* no address, look for MX record */

/*
 * Error return codes from getaddrinfo()
 */
#if 0
/* obsoleted */
#define	EAI_ADDRFAMILY	 1	/* address family for hostname not supported */
#endif
# 154 "/usr/include/netdb.h" 3 4
#define	EAI_AGAIN	 2	/* temporary failure in name resolution */
#define	EAI_BADFLAGS	 3	/* invalid value for ai_flags */
#define	EAI_FAIL	 4	/* non-recoverable failure in name resolution */
#define	EAI_FAMILY	 5	/* ai_family not supported */
#define	EAI_MEMORY	 6	/* memory allocation failure */
#if 0
/* obsoleted */
#define	EAI_NODATA	 7	/* no address associated with hostname */
#endif
# 163 "/usr/include/netdb.h" 3 4
#define	EAI_NONAME	 8	/* hostname nor servname provided, or not known */
#define	EAI_SERVICE	 9	/* servname not supported for ai_socktype */
#define	EAI_SOCKTYPE	10	/* ai_socktype not supported */
#define	EAI_SYSTEM	11	/* system error returned in errno */
#define	EAI_BADHINTS	12	/* invalid value for hints */
#define	EAI_PROTOCOL	13	/* resolved protocol is unknown */
#define	EAI_OVERFLOW	14	/* argument buffer overflow */
#define	EAI_MAX		15

/*
 * Flag values for getaddrinfo()
 */
#define	AI_PASSIVE	0x00000001 /* get address to use bind() */
#define	AI_CANONNAME	0x00000002 /* fill ai_canonname */
#define	AI_NUMERICHOST	0x00000004 /* prevent host name resolution */
#define	AI_NUMERICSERV	0x00000008 /* prevent service name resolution */
/* valid flags for addrinfo (not a standard def, apps should not use it) */
#define AI_MASK \
    (AI_PASSIVE | AI_CANONNAME | AI_NUMERICHOST | AI_NUMERICSERV | \
    AI_ADDRCONFIG | AI_ALL | AI_V4MAPPED)

#define	AI_ALL		0x00000100 /* IPv6 and IPv4-mapped (with AI_V4MAPPED) */
#define	AI_V4MAPPED_CFG	0x00000200 /* accept IPv4-mapped if kernel supports */
#define	AI_ADDRCONFIG	0x00000400 /* only if any address is assigned */
#define	AI_V4MAPPED	0x00000800 /* accept IPv4-mapped IPv6 address */
/* special recommended flags for getipnodebyname */
#define	AI_DEFAULT	(AI_V4MAPPED_CFG | AI_ADDRCONFIG)

/*
 * Constants for getnameinfo()
 */
#define	NI_MAXHOST	1025
#define	NI_MAXSERV	32

/*
 * Flag values for getnameinfo()
 */
#define	NI_NOFQDN	0x00000001
#define	NI_NUMERICHOST	0x00000002
#define	NI_NAMEREQD	0x00000004
#define	NI_NUMERICSERV	0x00000008
#define	NI_DGRAM	0x00000010
#if 0 /* obsolete */
#define NI_WITHSCOPEID	0x00000020
#endif
# 208 "/usr/include/netdb.h" 3 4

/*
 * Scope delimit character
 */
#define	SCOPE_DELIMITER	'%'

__BEGIN_DECLS
void		endhostent(void);
void		endnetent(void);
void		endprotoent(void);
void		endservent(void);
#if __BSD_VISIBLE || (__POSIX_VISIBLE && __POSIX_VISIBLE <= 200112)
struct hostent	*gethostbyaddr(const void *, socklen_t, int);
struct hostent	*gethostbyname(const char *);
#endif
# 223 "/usr/include/netdb.h" 3 4
struct hostent	*gethostent(void);
struct netent	*getnetbyaddr(uint32_t, int);
struct netent	*getnetbyname(const char *);
struct netent	*getnetent(void);
struct protoent	*getprotobyname(const char *);
struct protoent	*getprotobynumber(int);
struct protoent	*getprotoent(void);
struct servent	*getservbyname(const char *, const char *);
struct servent	*getservbyport(int, const char *);
struct servent	*getservent(void);
void		sethostent(int);
/* void		sethostfile(const char *); */
void		setnetent(int);
void		setprotoent(int);
int		getaddrinfo(const char *, const char *,
			    const struct addrinfo *, struct addrinfo **);
int		getnameinfo(const struct sockaddr *, socklen_t, char *,
			    size_t, char *, size_t, int);
void		freeaddrinfo(struct addrinfo *);
const char	*gai_strerror(int);
void		setservent(int);

#if __BSD_VISIBLE
void		endnetgrent(void);
void		freehostent(struct hostent *);
int		gethostbyaddr_r(const void *, socklen_t, int, struct hostent *,
    char *, size_t, struct hostent **, int *);
int		gethostbyname_r(const char *, struct hostent *, char *, size_t,
    struct hostent **, int *);
struct hostent	*gethostbyname2(const char *, int);
int		gethostbyname2_r(const char *, int, struct hostent *, char *,
    size_t, struct hostent **, int *);
int		gethostent_r(struct hostent *, char *, size_t,
    struct hostent **, int *);
struct hostent	*getipnodebyaddr(const void *, size_t, int, int *);
struct hostent	*getipnodebyname(const char *, int, int, int *);
int		getnetbyaddr_r(uint32_t, int, struct netent *, char *, size_t,
    struct netent**, int *);
int		getnetbyname_r(const char *, struct netent *, char *, size_t,
    struct netent **, int *);
int		getnetent_r(struct netent *, char *, size_t, struct netent **,
    int *);
int		getnetgrent(char **, char **, char **);
int		getprotobyname_r(const char *, struct protoent *, char *,
    size_t, struct protoent **);
int		getprotobynumber_r(int, struct protoent *, char *, size_t,
    struct protoent **);
int		getprotoent_r(struct protoent *, char *, size_t,
    struct protoent **);
int		getservbyname_r(const char *, const char *, struct servent *,
    char *, size_t, struct servent **);
int		getservbyport_r(int, const char *, struct servent *, char *,
    size_t, struct servent **);
int		getservent_r(struct servent *, char *, size_t,
    struct servent **);
void		herror(const char *);
const char	*hstrerror(int);
int		innetgr(const char *, const char *, const char *, const char *);
void		setnetgrent(const char *);
#endif
# 283 "/usr/include/netdb.h" 3 4


/*
 * PRIVATE functions specific to the FreeBSD implementation
 */

/* DO NOT USE THESE, THEY ARE SUBJECT TO CHANGE AND ARE NOT PORTABLE!!! */
int	* __h_errno(void);
__END_DECLS

#endif /* !_NETDB_H_ */
# 294 "/usr/include/netdb.h" 3 4
# 110 "sys/unix/erl_unix_sys.h" 2

#ifdef HAVE_POSIX_MEMALIGN
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 1
#endif
# 114 "sys/unix/erl_unix_sys.h"

/*
 * Make sure that MAXPATHLEN is defined.
 */
#ifndef MAXPATHLEN
#   ifdef PATH_MAX
#       define MAXPATHLEN PATH_MAX
#   else
# 122 "sys/unix/erl_unix_sys.h"
#       define MAXPATHLEN 2048
#   endif
# 124 "sys/unix/erl_unix_sys.h"
#endif
# 125 "sys/unix/erl_unix_sys.h"

/*
 * Min number of async threads
 */
#define  ERTS_MIN_NO_OF_ASYNC_THREADS 0

/* File descriptors are numbers anc consecutively allocated on Unix */
#define  ERTS_SYS_CONTINOUS_FD_NUMBERS

#ifndef ERTS_SMP
#  undef ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
#  define ERTS_POLL_NEED_ASYNC_INTERRUPT_SUPPORT
#endif
# 138 "sys/unix/erl_unix_sys.h"

#ifndef ENABLE_CHILD_WAITER_THREAD
#  ifdef ERTS_SMP
#    define ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
void erts_check_children(void);
#  endif
# 144 "sys/unix/erl_unix_sys.h"
#endif
# 145 "sys/unix/erl_unix_sys.h"

typedef void *GETENV_STATE;

/*
** For the erl_timer_sup module.
*/
typedef time_t erts_time_t;

typedef struct timeval SysTimeval;

#define sys_gettimeofday(Arg) ((void) gettimeofday((Arg), NULL))

typedef struct tms SysTimes;

#define SYS_CLK_TCK (erts_sys_time_data__.r.o.ticks_per_sec)

#define sys_times(Arg) times(Arg)

#if SIZEOF_LONG == 8
typedef long ErtsMonotonicTime;
typedef long ErtsSysHrTime;
#elif SIZEOF_LONG_LONG == 8
# 167 "sys/unix/erl_unix_sys.h"
typedef long long ErtsMonotonicTime;
typedef long long ErtsSysHrTime;
#else
# 170 "sys/unix/erl_unix_sys.h"
#error No signed 64-bit type found...
#endif
# 172 "sys/unix/erl_unix_sys.h"

typedef ErtsMonotonicTime ErtsSystemTime;

#define ERTS_MONOTONIC_TIME_MIN (((ErtsMonotonicTime) 1) << 63)
#define ERTS_MONOTONIC_TIME_MAX (~ERTS_MONOTONIC_TIME_MIN)

/*
 * OS monotonic time and OS system time
 */
#undef ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__

#if defined(OS_SYSTEM_TIME_USING_CLOCK_GETTIME) \
    && defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
#  if defined(__linux__)
#    define ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__ 1
#  endif
# 188 "sys/unix/erl_unix_sys.h"
#endif
# 189 "sys/unix/erl_unix_sys.h"

ErtsSystemTime erts_os_system_time(void);

#undef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#undef ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
#undef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__

#if defined(OS_MONOTONIC_TIME_USING_CLOCK_GETTIME)
#  define ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT 1
#  define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000*1000)
#  if defined(__linux__)
#    define ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__ 1
#  endif
# 202 "sys/unix/erl_unix_sys.h"
#elif defined(OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME)
# 203 "sys/unix/erl_unix_sys.h"
#  define ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT 1
#  define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000*1000)
#elif defined(OS_MONOTONIC_TIME_USING_GETHRTIME)
# 206 "sys/unix/erl_unix_sys.h"
#  define ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT 1
#  define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000*1000)
#elif defined(OS_MONOTONIC_TIME_USING_TIMES)
# 209 "sys/unix/erl_unix_sys.h"
#  define ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT 1
/* Time unit determined at runtime... */
#  define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT 0
#else /* No OS monotonic available... */
# 213 "sys/unix/erl_unix_sys.h"
#  define ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT (1000*1000)
#endif
# 215 "sys/unix/erl_unix_sys.h"

/*
 * erts_sys_hrtime() is the highest resolution
 * time function found. Time unit is nano-seconds.
 * It may or may not be monotonic.
 */
ErtsSysHrTime erts_sys_hrtime(void);

struct erts_sys_time_read_only_data__ {
#ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
    ErtsMonotonicTime (*os_monotonic_time)(void);
#endif
# 227 "sys/unix/erl_unix_sys.h"
#ifdef ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__
    void (*os_times)(ErtsMonotonicTime *, ErtsSystemTime *);
#endif
# 230 "sys/unix/erl_unix_sys.h"
    int ticks_per_sec;
};

typedef struct {
    union {
	struct erts_sys_time_read_only_data__ o;
	char align__[(((sizeof(struct erts_sys_time_read_only_data__) - 1)
		       / ASSUMED_CACHE_LINE_SIZE) + 1)
		     * ASSUMED_CACHE_LINE_SIZE];
    } r;
} ErtsSysTimeData__;

extern ErtsSysTimeData__ erts_sys_time_data__;

#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT

#ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__
ERTS_GLB_INLINE
#endif
# 249 "sys/unix/erl_unix_sys.h"
ErtsMonotonicTime erts_os_monotonic_time(void);

#ifdef ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__
ERTS_GLB_INLINE
#endif
# 254 "sys/unix/erl_unix_sys.h"
void erts_os_times(ErtsMonotonicTime *, ErtsSystemTime *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

#ifdef ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__

ERTS_GLB_INLINE ErtsMonotonicTime
erts_os_monotonic_time(void)
{
    return (*erts_sys_time_data__.r.o.os_monotonic_time)();
}

#endif /* ERTS_OS_MONOTONIC_INLINE_FUNC_PTR_CALL__ */
# 267 "sys/unix/erl_unix_sys.h"

#ifdef ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__

ERTS_GLB_INLINE void
erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep)
{
    return (*erts_sys_time_data__.r.o.os_times)(mtimep, stimep);
}

#endif /* ERTS_OS_TIMES_INLINE_FUNC_PTR_CALL__ */
# 277 "sys/unix/erl_unix_sys.h"

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 279 "sys/unix/erl_unix_sys.h"

#endif /* ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT */
# 281 "sys/unix/erl_unix_sys.h"

/*
 *
 */

#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
typedef long long SysCpuTime;
typedef struct timespec SysTimespec;

#if defined(HAVE_GETHRVTIME)
#define sys_gethrvtime() gethrvtime()
#define sys_get_proc_cputime(t,tp) (t) = sys_gethrvtime(), \
                                   (tp).tv_sec = (time_t)((t)/1000000000LL), \
                                   (tp).tv_nsec = (long)((t)%1000000000LL)
int sys_start_hrvtime(void);
int sys_stop_hrvtime(void);

#elif defined(HAVE_CLOCK_GETTIME_CPU_TIME)
# 299 "sys/unix/erl_unix_sys.h"
#define sys_clock_gettime(cid,tp) clock_gettime((cid),&(tp))
#define sys_get_proc_cputime(t,tp) sys_clock_gettime(CLOCK_PROCESS_CPUTIME_ID,(tp))

#endif
# 303 "sys/unix/erl_unix_sys.h"
#endif
# 304 "sys/unix/erl_unix_sys.h"

/* No use in having other resolutions than 1 Ms. */
#define SYS_CLOCK_RESOLUTION 1

/* These are defined in sys.c */
typedef void (*SIGFUNC)(int);
extern SIGFUNC sys_signal(int, SIGFUNC);
extern void sys_sigrelease(int);
extern void sys_sigblock(int);
extern void sys_stop_cat(void);

/*
 * Handling of floating point exceptions.
 */

#ifdef USE_ISINF_ISNAN		/* simulate finite() */
#  define isfinite(f) (!isinf(f) && !isnan(f))
#  define HAVE_ISFINITE
#elif (defined(__GNUC__) && !defined(__llvm__)) && defined(HAVE_FINITE)
# 323 "sys/unix/erl_unix_sys.h"
/* We use finite in gcc as it emits assembler instead of
   the function call that isfinite emits. The assembler is
   significantly faster. */
#  ifdef isfinite
#     undef isfinite
#  endif
# 329 "sys/unix/erl_unix_sys.h"
#  define isfinite finite
#  ifndef HAVE_ISFINITE
#    define HAVE_ISFINITE
#  endif
# 333 "sys/unix/erl_unix_sys.h"
#elif defined(isfinite) && !defined(HAVE_ISFINITE)
# 334 "sys/unix/erl_unix_sys.h"
#  define HAVE_ISFINITE
#elif !defined(HAVE_ISFINITE) && defined(HAVE_FINITE)
# 336 "sys/unix/erl_unix_sys.h"
#  define isfinite finite
#  define HAVE_ISFINITE
#endif
# 339 "sys/unix/erl_unix_sys.h"

#define erts_isfinite isfinite

#ifdef NO_FPE_SIGNALS

#define erts_get_current_fp_exception() NULL
#ifdef ERTS_SMP
#define erts_thread_init_fp_exception() do{}while(0)
#endif
# 348 "sys/unix/erl_unix_sys.h"
#  define __ERTS_FP_CHECK_INIT(fpexnp) do {} while (0)
#  define __ERTS_FP_ERROR(fpexnp, f, Action) if (!isfinite(f)) { Action; } else {}
#  define __ERTS_FP_ERROR_THOROUGH(fpexnp, f, Action) __ERTS_FP_ERROR(fpexnp, f, Action)
#  define __ERTS_SAVE_FP_EXCEPTION(fpexnp)
#  define __ERTS_RESTORE_FP_EXCEPTION(fpexnp)

#define erts_sys_block_fpe() 0
#define erts_sys_unblock_fpe(x) do{}while(0)

#else /* !NO_FPE_SIGNALS */
# 358 "sys/unix/erl_unix_sys.h"

extern volatile unsigned long *erts_get_current_fp_exception(void);
#ifdef ERTS_SMP
extern void erts_thread_init_fp_exception(void);
#endif
# 363 "sys/unix/erl_unix_sys.h"
#  if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__)
#    define erts_fwait(fpexnp,f) \
	__asm__ __volatile__("fwait" : "=m"(*(fpexnp)) : "m"(f))
#  elif (defined(__powerpc__) || defined(__ppc__)) && defined(__GNUC__)
# 367 "sys/unix/erl_unix_sys.h"
#    define erts_fwait(fpexnp,f) \
	__asm__ __volatile__("" : "=m"(*(fpexnp)) : "fm"(f))
#  elif defined(__sparc__) && defined(__linux__) && defined(__GNUC__)
# 370 "sys/unix/erl_unix_sys.h"
#    define erts_fwait(fpexnp,f) \
	__asm__ __volatile__("" : "=m"(*(fpexnp)) : "em"(f))
#  else
# 373 "sys/unix/erl_unix_sys.h"
#    define erts_fwait(fpexnp,f) \
	__asm__ __volatile__("" : "=m"(*(fpexnp)) : "g"(f))
#  endif
# 376 "sys/unix/erl_unix_sys.h"
#  if (defined(__i386__) || defined(__x86_64__)) && defined(__GNUC__)
     extern void erts_restore_fpu(void);
#  else
# 379 "sys/unix/erl_unix_sys.h"
#    define erts_restore_fpu() /*empty*/
#  endif
# 381 "sys/unix/erl_unix_sys.h"
#  if (!defined(__GNUC__) || \
       (__GNUC__ < 2) || \
       (__GNUC__ == 2 && __GNUC_MINOR < 96)) && \
      !defined(__builtin_expect)
#    define __builtin_expect(x, expected_value) (x)
#  endif
# 387 "sys/unix/erl_unix_sys.h"
static __inline__ int erts_check_fpe(volatile unsigned long *fp_exception, double f)
{
    erts_fwait(fp_exception, f);
    if (__builtin_expect(*fp_exception == 0, 1))
       return 0;
    *fp_exception = 0;
    erts_restore_fpu();
    return 1;
}
#  undef erts_fwait
#  undef erts_restore_fpu
extern void erts_fp_check_init_error(volatile unsigned long *fp_exception);
static __inline__ void __ERTS_FP_CHECK_INIT(volatile unsigned long *fp_exception)
{
    if (__builtin_expect(*fp_exception == 0, 1))
	return;
    erts_fp_check_init_error(fp_exception);
}
#  define __ERTS_FP_ERROR(fpexnp, f, Action) do { if (erts_check_fpe((fpexnp),(f))) { Action; } } while (0)
#  define __ERTS_SAVE_FP_EXCEPTION(fpexnp) unsigned long old_erl_fp_exception = *(fpexnp)
#  define __ERTS_RESTORE_FP_EXCEPTION(fpexnp) \
              do { *(fpexnp) = old_erl_fp_exception; } while (0)
   /* This is for library calls where we don't trust the external
      code to always throw floating-point exceptions on errors. */
static __inline__ int erts_check_fpe_thorough(volatile unsigned long *fp_exception, double f)
{
    return erts_check_fpe(fp_exception, f) || !isfinite(f);
}
#  define __ERTS_FP_ERROR_THOROUGH(fpexnp, f, Action) \
  do { if (erts_check_fpe_thorough((fpexnp),(f))) { Action; } } while (0)

int erts_sys_block_fpe(void);
void erts_sys_unblock_fpe(int);

#endif /* !NO_FPE_SIGNALS */
# 422 "sys/unix/erl_unix_sys.h"

#define ERTS_FP_CHECK_INIT(p)		__ERTS_FP_CHECK_INIT(&(p)->fp_exception)
#define ERTS_FP_ERROR(p, f, A)		__ERTS_FP_ERROR(&(p)->fp_exception, f, A)
#define ERTS_FP_ERROR_THOROUGH(p, f, A)	__ERTS_FP_ERROR_THOROUGH(&(p)->fp_exception, f, A)


#ifdef NEED_CHILD_SETUP_DEFINES
/* The child setup argv[] */
#define CS_ARGV_PROGNAME_IX	0		/* Program name		*/
#define CS_ARGV_UNBIND_IX	1		/* Unbind from cpu	*/
#define CS_ARGV_WD_IX		2		/* Working directory	*/
#define CS_ARGV_CMD_IX		3		/* Command		*/
#define CS_ARGV_FD_CR_IX	4		/* Fd close range	*/
#define CS_ARGV_DUP2_OP_IX(N)	((N) + 5)	/* dup2 operations	*/

#define CS_ARGV_NO_OF_DUP2_OPS	3		/* Number of dup2 ops	*/
#define CS_ARGV_NO_OF_ARGS	8		/* Number of arguments	*/
#endif /* #ifdef NEED_CHILD_SETUP_DEFINES */
# 440 "sys/unix/erl_unix_sys.h"

/* Threads */
#ifdef USE_THREADS
extern int init_async(int);
extern int exit_async(void);
#endif
# 446 "sys/unix/erl_unix_sys.h"

#define ERTS_EXIT_AFTER_DUMP _exit

#if !defined(__APPLE__) && !defined(__MACH__)
/* Some OS X versions do not allow (ab)using signal handlers like this */
#define ERTS_HAVE_TRY_CATCH 1

/* We try to simulate a try catch in C with the help of signal handlers.
 * Only use this as a very last resort, as it is not very portable and
 * quite unstable. It is also not thread safe, so make sure that only
 * one thread can call this at a time!
 */
extern void erts_sys_sigsegv_handler(int);
extern jmp_buf erts_sys_sigsegv_jmp;
#define ERTS_SYS_TRY_CATCH(EXPR,CATCH)                                  \
    do {                                                                \
        SIGFUNC prev_handler = sys_signal(SIGSEGV,                      \
                                          erts_sys_sigsegv_handler);    \
        if (!setjmp(erts_sys_sigsegv_jmp)) {                            \
            EXPR;                                                       \
        } else {                                                        \
            CATCH;                                                      \
        }                                                               \
        sys_signal(SIGSEGV,prev_handler);                               \
    } while(0)
#endif
# 472 "sys/unix/erl_unix_sys.h"

#endif /* #ifndef _ERL_UNIX_SYS_H */
# 474 "sys/unix/erl_unix_sys.h"
# 81 "beam/sys.h" 2
#ifndef UNIX
#  define UNIX 1
#endif
# 84 "beam/sys.h"
#endif
# 85 "beam/sys.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_misc_utils.h"
#endif /* expanded by -frewrite-includes */
# 86 "beam/sys.h"
# 1 "../include/internal/erl_misc_utils.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2006-2010. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef ERL_MISC_UTILS_H_
#define ERL_MISC_UTILS_H_

#if 0 /* expanded by -frewrite-includes */
#include "erl_errno.h"
#endif /* expanded by -frewrite-includes */
# 24 "../include/internal/erl_misc_utils.h"
# 25 "../include/internal/erl_misc_utils.h"

typedef struct erts_cpu_info_t_ erts_cpu_info_t;
typedef struct {
    int node;
    int processor;
    int processor_node;
    int core;
    int thread;
    int logical;
} erts_cpu_topology_t;

erts_cpu_info_t *erts_cpu_info_create(void);
void erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo);
int erts_cpu_info_update(erts_cpu_info_t *cpuinfo);
int erts_get_cpu_configured(erts_cpu_info_t *cpuinfo);
int erts_get_cpu_online(erts_cpu_info_t *cpuinfo);
int erts_get_cpu_available(erts_cpu_info_t *cpuinfo);
char *erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo);
int erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no);
int erts_get_cpu_topology_size(erts_cpu_info_t *cpuinfo);
int erts_get_cpu_topology(erts_cpu_info_t *cpuinfo,
			  erts_cpu_topology_t *topology);
int erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id);
int erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu);
int erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo);
int erts_unbind_from_cpu_str(char *str);

int erts_milli_sleep(long);

#ifdef __WIN32__
int erts_map_win_error_to_errno(DWORD win_error);
int erts_get_last_win_errno(void);
#endif
# 58 "../include/internal/erl_misc_utils.h"

#endif /* #ifndef ERL_MISC_UTILS_H_ */
# 60 "../include/internal/erl_misc_utils.h"
# 87 "beam/sys.h" 2

/*
 * To allow building of Universal Binaries for Mac OS X,
 * we must not depend on the endian detected by the configure script.
 */
#if defined(__APPLE__)
#  if defined(__BIG_ENDIAN__) && !defined(WORDS_BIGENDIAN)
#    define WORDS_BIGENDIAN 1
#  elif !defined(__BIG_ENDIAN__) && defined(WORDS_BIGENDIAN)
# 96 "beam/sys.h"
#    undef WORDS_BIGENDIAN
#  endif
# 98 "beam/sys.h"
#endif
# 99 "beam/sys.h"

/*
 * Make sure we have a type for FD's (used by erl_check_io)
 */

#ifndef ERTS_SYS_FD_TYPE
#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
typedef int ErtsSysFdType;
#else
# 108 "beam/sys.h"
#ifndef ERTS_SYS_FD_INVALID
# error missing ERTS_SYS_FD_INVALID
#endif
# 111 "beam/sys.h"
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
# 113 "beam/sys.h"

#if !defined(__GNUC__)
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 117 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 120 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 123 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 126 "beam/sys.h"

#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
#  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
#  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# 131 "beam/sys.h"
#  define ERTS_LIKELY(BOOL)   (BOOL)
#  define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
# 134 "beam/sys.h"
#ifdef __GNUC__
#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
#  else
# 138 "beam/sys.h"
#    define ERTS_DECLARE_DUMMY(X) X
#  endif
# 140 "beam/sys.h"
#else
# 141 "beam/sys.h"
#  define ERTS_DECLARE_DUMMY(X) X
#endif
# 143 "beam/sys.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 149 "beam/sys.h"
#      define __func__ __FUNCTION__
#    endif
# 151 "beam/sys.h"
#  endif
# 152 "beam/sys.h"
#endif
# 153 "beam/sys.h"

#define ERTS_MK_VSN_INT(Major, Minor, Build) \
    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))

#ifndef ERTS_EXIT_AFTER_DUMP
#  define ERTS_EXIT_AFTER_DUMP exit
#endif
# 160 "beam/sys.h"

/* In VC++, noreturn is a declspec that has to be before the types,
 * but in GNUC it is an att ribute to be placed between return type
 * and function name, hence __decl_noreturn <types> __noreturn <function name>
 *
 * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
 */
#if __GNUC__
#  define __decl_noreturn
#  ifndef __noreturn
#     define __noreturn __attribute__((noreturn))
#  endif
# 172 "beam/sys.h"
#else
# 173 "beam/sys.h"
#  if defined(__WIN32__) && defined(_MSC_VER)
#    define __noreturn
#    define __decl_noreturn __declspec(noreturn)
#  else
# 177 "beam/sys.h"
#    define __noreturn
#    define __decl_noreturn
#  endif
# 180 "beam/sys.h"
#endif
# 181 "beam/sys.h"

#define ERTS_ASSERT(e) \
    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))

__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
						 const char* file, int line);

#ifdef DEBUG
#  define ASSERT(e) ERTS_ASSERT(e)
#else
# 191 "beam/sys.h"
#  define ASSERT(e) ((void) 1)
#endif
# 193 "beam/sys.h"

/* ERTS_UNDEF can be used to silence false warnings about
 * "variable may be used uninitialized" while keeping the variable
 * marked as undefined by valgrind.
 */
#ifdef VALGRIND
#  define ERTS_UNDEF(V,I)
#else
# 201 "beam/sys.h"
#  define ERTS_UNDEF(V,I) V = I
#endif
# 203 "beam/sys.h"

/*
 * Compile time assert
 * (the actual compiler error msg can be a bit confusing)
 */
#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
# define ERTS_CT_ASSERT(e) \
    do { \
	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
    } while(0)
#else
# 214 "beam/sys.h"
# define ERTS_CT_ASSERT(e) \
    do { \
        enum { compile_time_assert__ = 1/(e) }; \
    } while (0)
#endif
# 219 "beam/sys.h"

/*
 * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
 * But MSC doesn't define __STDC__, unless we compile with the -Za
 * flag (strict ANSI C, no Microsoft extension).  Compiling with -Za
 * doesn't work: some Microsoft headers fail to compile...
 *
 * Solution: Test if __STDC__ or _MSC_VER is defined.
 *
 * Note: Simply defining __STDC__ doesn't work, as some Microsoft
 * headers will fail to compile!
 */

#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 232 "beam/sys.h"
# 1 "/usr/include/stdarg.h" 1 3 4
/*-
 * Copyright (c) 2002 David E. O'Brien.  All rights reserved.
 * Copyright (c) 1991, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)stdarg.h	8.1 (Berkeley) 6/10/93
 * $FreeBSD: head/sys/arm/include/stdarg.h 162487 2006-09-21 01:37:02Z kan $
 */

#ifndef _MACHINE_STDARG_H_
#define	_MACHINE_STDARG_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/stdarg.h" 3 4
# 38 "/usr/include/stdarg.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/stdarg.h" 3 4
# 39 "/usr/include/stdarg.h" 3 4

#ifndef _VA_LIST_DECLARED
#define	_VA_LIST_DECLARED
typedef	__va_list	va_list;
#endif
# 44 "/usr/include/stdarg.h" 3 4

#ifdef __GNUCLIKE_BUILTIN_STDARG

#define	va_start(ap, last) \
	__builtin_va_start((ap), (last))

#define	va_arg(ap, type) \
	__builtin_va_arg((ap), type)

#if __ISO_C_VISIBLE >= 1999
#define	va_copy(dest, src) \
	__builtin_va_copy((dest), (src))
#endif
# 57 "/usr/include/stdarg.h" 3 4

#define	va_end(ap) \
	__builtin_va_end(ap)

#else	/* !__GNUCLIKE_BUILTIN_STDARG */
# 62 "/usr/include/stdarg.h" 3 4

#define	__va_size(type) \
	(((sizeof(type) + sizeof(int) - 1) / sizeof(int)) * sizeof(int))

#ifdef __GNUCLIKE_BUILTIN_NEXT_ARG
#define va_start(ap, last) \
	((ap) = (va_list)__builtin_next_arg(last))
#else	/* !__GNUCLIKE_BUILTIN_NEXT_ARG */
# 70 "/usr/include/stdarg.h" 3 4
#define	va_start(ap, last) \
	((ap) = (va_list)&(last) + __va_size(last))
#endif	/* __GNUCLIKE_BUILTIN_NEXT_ARG */
# 73 "/usr/include/stdarg.h" 3 4

#define	va_arg(ap, type) \
	(*(type *)((ap) += __va_size(type), (ap) - __va_size(type)))

#define	va_end(ap)

#endif /* __GNUCLIKE_BUILTIN_STDARG */
# 80 "/usr/include/stdarg.h" 3 4

#endif /* !_MACHINE_STDARG_H_ */
# 82 "/usr/include/stdarg.h" 3 4
# 233 "beam/sys.h" 2

/* This isn't sys-dependent, but putting it here benefits sys.c and drivers
   - allow use of 'const' regardless of compiler */

#if !defined(__STDC__) && !defined(_MSC_VER)
#  define const
#endif
# 240 "beam/sys.h"

#undef __deprecated
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
#  define __deprecated __attribute__((deprecated))
#else
# 245 "beam/sys.h"
#  define __deprecated
#endif
# 247 "beam/sys.h"
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
#  define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
#else
# 250 "beam/sys.h"
#  define erts_align_attribute(SZ)
#endif
# 252 "beam/sys.h"

/*
** Data types:
**
** Eterm: A tagged erlang term (possibly 64 bits)
** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
** UInt:  An unsigned integer exactly as large as an Eterm.
** SInt:  A signed integer exactly as large as an eterm and therefor large
**        enough to hold the return value of the signed_val() macro.
** UWord: An unsigned integer at least as large as a void * and also as large
**          or larger than an Eterm
** SWord: A signed integer at least as large as a void * and also as large
**          or larger than an Eterm
** Uint32: An unsigned integer of 32 bits exactly
** Sint32: A signed integer of 32 bits exactly
** Uint16: An unsigned integer of 16 bits exactly
** Sint16: A signed integer of 16 bits exactly.
*/

#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
#error Cannot handle this combination of int/long/void*/size_t sizes
#endif
# 276 "beam/sys.h"

#if SIZEOF_VOID_P == 8
#undef  ARCH_32
#define ARCH_64
#define ERTS_SIZEOF_TERM 8
#elif SIZEOF_VOID_P == 4
# 282 "beam/sys.h"
#define ARCH_32
#undef  ARCH_64
#define ERTS_SIZEOF_TERM 4
#else
# 286 "beam/sys.h"
#error Neither 32 nor 64 bit architecture
#endif
# 288 "beam/sys.h"
#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
#    define HALFWORD_HEAP 1
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND) ASSERT(COND)
#    undef ERTS_SIZEOF_TERM
#    define ERTS_SIZEOF_TERM 4
#else
# 295 "beam/sys.h"
#    define HALFWORD_HEAP 0
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND)
#endif
# 299 "beam/sys.h"

#if SIZEOF_VOID_P != SIZEOF_SIZE_T
#error sizeof(void*) != sizeof(size_t)
#endif
# 303 "beam/sys.h"

#if HALFWORD_HEAP

#if SIZEOF_INT == 4
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define ERTS_UINT_MAX UINT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#else
# 314 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 316 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long UWord;
typedef long          SWord;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#elif SIZEOF_VOID_P == SIZEOF_INT
# 325 "beam/sys.h"
typedef unsigned int UWord;
typedef int          SWord;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 332 "beam/sys.h"
typedef unsigned long long UWord;
typedef long long          SWord;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#else
# 339 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 341 "beam/sys.h"

#else /* !HALFWORD_HEAP */
# 343 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long Eterm;
typedef unsigned long Uint;
typedef long          Sint;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_INT
# 355 "beam/sys.h"
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 365 "beam/sys.h"
typedef unsigned long long Eterm;
typedef unsigned long long Uint;
typedef long long          Sint;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
#if defined(__WIN32__)
#define ErtsStrToSint _strtoi64
#else
# 376 "beam/sys.h"
#define ErtsStrToSint strtoll
#endif
# 378 "beam/sys.h"
#else
# 379 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 381 "beam/sys.h"

typedef Uint UWord;
typedef Sint SWord;
#define ERTS_UINT_MAX ERTS_UWORD_MAX

#endif /* HALFWORD_HEAP */
# 387 "beam/sys.h"

typedef UWord BeamInstr;

#ifndef HAVE_INT64
#  if SIZEOF_LONG == 8
#    define HAVE_INT64 1
typedef unsigned long Uint64;
typedef long          Sint64;
#    ifdef ULONG_MAX
#      define ERTS_UINT64_MAX ULONG_MAX
#    endif
# 398 "beam/sys.h"
#    ifdef LONG_MAX
#      define ERTS_SINT64_MAX LONG_MAX
#    endif
# 401 "beam/sys.h"
#    ifdef LONG_MIN
#      define ERTS_SINT64_MIN LONG_MIN
#    endif
# 404 "beam/sys.h"
#  elif SIZEOF_LONG_LONG == 8
# 405 "beam/sys.h"
#    define HAVE_INT64 1
typedef unsigned long long Uint64;
typedef long long          Sint64;
#    ifdef ULLONG_MAX
#      define ERTS_UINT64_MAX ULLONG_MAX
#    endif
# 411 "beam/sys.h"
#    ifdef LLONG_MAX
#      define ERTS_SINT64_MAX LLONG_MAX
#    endif
# 414 "beam/sys.h"
#    ifdef LLONG_MIN
#      define ERTS_SINT64_MIN LLONG_MIN
#    endif
# 417 "beam/sys.h"
#  else
# 418 "beam/sys.h"
#    error "No 64-bit integer type found"
#  endif
# 420 "beam/sys.h"
#endif
# 421 "beam/sys.h"

#ifndef ERTS_UINT64_MAX
#  define ERTS_UINT64_MAX (~((Uint64) 0))
#endif
# 425 "beam/sys.h"
#ifndef ERTS_SINT64_MAX
#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
#endif
# 428 "beam/sys.h"
#ifndef ERTS_SINT64_MIN
#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
#endif
# 431 "beam/sys.h"

#if SIZEOF_LONG == 4
typedef unsigned long Uint32;
typedef long          Sint32;
#elif SIZEOF_INT == 4
# 436 "beam/sys.h"
typedef unsigned int Uint32;
typedef int          Sint32;
#else
# 439 "beam/sys.h"
#error Found no appropriate type to use for 'Uint32' and 'Sint32'
#endif
# 441 "beam/sys.h"

#if SIZEOF_INT == 2
typedef unsigned int Uint16;
typedef int          Sint16;
#elif SIZEOF_SHORT == 2
# 446 "beam/sys.h"
typedef unsigned short Uint16;
typedef short          Sint16;
#else
# 449 "beam/sys.h"
#error Found no appropriate type to use for 'Uint16' and 'Sint16'
#endif
# 451 "beam/sys.h"

#if CHAR_BIT == 8
typedef unsigned char byte;
#else
# 455 "beam/sys.h"
#error Found no appropriate type to use for 'byte'
#endif
# 457 "beam/sys.h"

#if defined(ARCH_64) && !HAVE_INT64
#error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
#endif
# 461 "beam/sys.h"

#ifdef WORDS_BIGENDIAN
#  define ERTS_HUINT_HVAL_HIGH 0
#  define ERTS_HUINT_HVAL_LOW 1
#else
# 466 "beam/sys.h"
#  define ERTS_HUINT_HVAL_HIGH 1
#  define ERTS_HUINT_HVAL_LOW 0
#endif
# 469 "beam/sys.h"
#if ERTS_SIZEOF_TERM == 8
typedef union {
    Uint val;
    Uint32 hval[2];
} HUint;
#elif ERTS_SIZEOF_TERM == 4
# 475 "beam/sys.h"
typedef union {
    Uint val;
    Uint16 hval[2];
} HUint;
#else
# 480 "beam/sys.h"
#error "Unsupported size of term"
#endif
# 482 "beam/sys.h"

#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
    (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 486 "beam/sys.h"
# 1 "beam/erl_lock_check.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2005-2012. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
 * Description: A lock checker that checks that each thread acquires
 *              locks according to a predefined global lock order. The
 *              global lock order is used to prevent deadlocks. If the
 *              lock order is violated, an error message is printed
 *              and the emulator aborts. The lock checker is only
 *              intended to be enabled when debugging.
 *
 * Author: Rickard Green
 */

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_lock_check.h"
# 1 "beam/sys.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __SYS_H__
#define __SYS_H__

#ifdef ERTS_INLINE
#  ifndef ERTS_CAN_INLINE
#    define ERTS_CAN_INLINE 1
#  endif
# 28 "beam/sys.h"
#else
# 29 "beam/sys.h"
#  if defined(__GNUC__)
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline__
#  elif defined(__WIN32__)
# 33 "beam/sys.h"
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline
#  else
# 36 "beam/sys.h"
#    define ERTS_CAN_INLINE 0
#    define ERTS_INLINE
#  endif
# 39 "beam/sys.h"
#endif
# 40 "beam/sys.h"

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#  undef ERTS_CAN_INLINE
#  define ERTS_CAN_INLINE 0
#  undef ERTS_INLINE
#  define ERTS_INLINE
#endif
# 47 "beam/sys.h"

#if ERTS_CAN_INLINE
#define ERTS_GLB_INLINE static ERTS_INLINE
#else
# 51 "beam/sys.h"
#define ERTS_GLB_INLINE
#endif
# 53 "beam/sys.h"

#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
#else
# 57 "beam/sys.h"
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
# 59 "beam/sys.h"

#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
#  define NO_FPE_SIGNALS
#endif
# 63 "beam/sys.h"

#ifdef DISABLE_CHILD_WAITER_THREAD
#undef ENABLE_CHILD_WAITER_THREAD
#endif
# 67 "beam/sys.h"

#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
#undef ENABLE_CHILD_WAITER_THREAD
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
# 72 "beam/sys.h"

#define ERTS_I64_LITERAL(X) X##LL

#if defined (__WIN32__)
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_sys.h"
#endif /* expanded by -frewrite-includes */
# 76 "beam/sys.h"
# 77 "beam/sys.h"
#elif defined (__OSE__)
# 78 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_ose_sys.h"
#endif /* expanded by -frewrite-includes */
# 78 "beam/sys.h"
# 79 "beam/sys.h"
#else 
# 80 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_unix_sys.h"
#endif /* expanded by -frewrite-includes */
# 80 "beam/sys.h"
# 81 "beam/sys.h"
#ifndef UNIX
#  define UNIX 1
#endif
# 84 "beam/sys.h"
#endif
# 85 "beam/sys.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_misc_utils.h"
#endif /* expanded by -frewrite-includes */
# 86 "beam/sys.h"
# 87 "beam/sys.h"

/*
 * To allow building of Universal Binaries for Mac OS X,
 * we must not depend on the endian detected by the configure script.
 */
#if defined(__APPLE__)
#  if defined(__BIG_ENDIAN__) && !defined(WORDS_BIGENDIAN)
#    define WORDS_BIGENDIAN 1
#  elif !defined(__BIG_ENDIAN__) && defined(WORDS_BIGENDIAN)
# 96 "beam/sys.h"
#    undef WORDS_BIGENDIAN
#  endif
# 98 "beam/sys.h"
#endif
# 99 "beam/sys.h"

/*
 * Make sure we have a type for FD's (used by erl_check_io)
 */

#ifndef ERTS_SYS_FD_TYPE
#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
typedef int ErtsSysFdType;
#else
# 108 "beam/sys.h"
#ifndef ERTS_SYS_FD_INVALID
# error missing ERTS_SYS_FD_INVALID
#endif
# 111 "beam/sys.h"
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
# 113 "beam/sys.h"

#if !defined(__GNUC__)
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 117 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 120 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 123 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 126 "beam/sys.h"

#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
#  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
#  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# 131 "beam/sys.h"
#  define ERTS_LIKELY(BOOL)   (BOOL)
#  define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
# 134 "beam/sys.h"
#ifdef __GNUC__
#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
#  else
# 138 "beam/sys.h"
#    define ERTS_DECLARE_DUMMY(X) X
#  endif
# 140 "beam/sys.h"
#else
# 141 "beam/sys.h"
#  define ERTS_DECLARE_DUMMY(X) X
#endif
# 143 "beam/sys.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 149 "beam/sys.h"
#      define __func__ __FUNCTION__
#    endif
# 151 "beam/sys.h"
#  endif
# 152 "beam/sys.h"
#endif
# 153 "beam/sys.h"

#define ERTS_MK_VSN_INT(Major, Minor, Build) \
    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))

#ifndef ERTS_EXIT_AFTER_DUMP
#  define ERTS_EXIT_AFTER_DUMP exit
#endif
# 160 "beam/sys.h"

/* In VC++, noreturn is a declspec that has to be before the types,
 * but in GNUC it is an att ribute to be placed between return type
 * and function name, hence __decl_noreturn <types> __noreturn <function name>
 *
 * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
 */
#if __GNUC__
#  define __decl_noreturn
#  ifndef __noreturn
#     define __noreturn __attribute__((noreturn))
#  endif
# 172 "beam/sys.h"
#else
# 173 "beam/sys.h"
#  if defined(__WIN32__) && defined(_MSC_VER)
#    define __noreturn
#    define __decl_noreturn __declspec(noreturn)
#  else
# 177 "beam/sys.h"
#    define __noreturn
#    define __decl_noreturn
#  endif
# 180 "beam/sys.h"
#endif
# 181 "beam/sys.h"

#define ERTS_ASSERT(e) \
    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))

__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
						 const char* file, int line);

#ifdef DEBUG
#  define ASSERT(e) ERTS_ASSERT(e)
#else
# 191 "beam/sys.h"
#  define ASSERT(e) ((void) 1)
#endif
# 193 "beam/sys.h"

/* ERTS_UNDEF can be used to silence false warnings about
 * "variable may be used uninitialized" while keeping the variable
 * marked as undefined by valgrind.
 */
#ifdef VALGRIND
#  define ERTS_UNDEF(V,I)
#else
# 201 "beam/sys.h"
#  define ERTS_UNDEF(V,I) V = I
#endif
# 203 "beam/sys.h"

/*
 * Compile time assert
 * (the actual compiler error msg can be a bit confusing)
 */
#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
# define ERTS_CT_ASSERT(e) \
    do { \
	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
    } while(0)
#else
# 214 "beam/sys.h"
# define ERTS_CT_ASSERT(e) \
    do { \
        enum { compile_time_assert__ = 1/(e) }; \
    } while (0)
#endif
# 219 "beam/sys.h"

/*
 * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
 * But MSC doesn't define __STDC__, unless we compile with the -Za
 * flag (strict ANSI C, no Microsoft extension).  Compiling with -Za
 * doesn't work: some Microsoft headers fail to compile...
 *
 * Solution: Test if __STDC__ or _MSC_VER is defined.
 *
 * Note: Simply defining __STDC__ doesn't work, as some Microsoft
 * headers will fail to compile!
 */

#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 232 "beam/sys.h"
# 233 "beam/sys.h"

/* This isn't sys-dependent, but putting it here benefits sys.c and drivers
   - allow use of 'const' regardless of compiler */

#if !defined(__STDC__) && !defined(_MSC_VER)
#  define const
#endif
# 240 "beam/sys.h"

#undef __deprecated
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
#  define __deprecated __attribute__((deprecated))
#else
# 245 "beam/sys.h"
#  define __deprecated
#endif
# 247 "beam/sys.h"
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
#  define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
#else
# 250 "beam/sys.h"
#  define erts_align_attribute(SZ)
#endif
# 252 "beam/sys.h"

/*
** Data types:
**
** Eterm: A tagged erlang term (possibly 64 bits)
** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
** UInt:  An unsigned integer exactly as large as an Eterm.
** SInt:  A signed integer exactly as large as an eterm and therefor large
**        enough to hold the return value of the signed_val() macro.
** UWord: An unsigned integer at least as large as a void * and also as large
**          or larger than an Eterm
** SWord: A signed integer at least as large as a void * and also as large
**          or larger than an Eterm
** Uint32: An unsigned integer of 32 bits exactly
** Sint32: A signed integer of 32 bits exactly
** Uint16: An unsigned integer of 16 bits exactly
** Sint16: A signed integer of 16 bits exactly.
*/

#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
#error Cannot handle this combination of int/long/void*/size_t sizes
#endif
# 276 "beam/sys.h"

#if SIZEOF_VOID_P == 8
#undef  ARCH_32
#define ARCH_64
#define ERTS_SIZEOF_TERM 8
#elif SIZEOF_VOID_P == 4
# 282 "beam/sys.h"
#define ARCH_32
#undef  ARCH_64
#define ERTS_SIZEOF_TERM 4
#else
# 286 "beam/sys.h"
#error Neither 32 nor 64 bit architecture
#endif
# 288 "beam/sys.h"
#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
#    define HALFWORD_HEAP 1
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND) ASSERT(COND)
#    undef ERTS_SIZEOF_TERM
#    define ERTS_SIZEOF_TERM 4
#else
# 295 "beam/sys.h"
#    define HALFWORD_HEAP 0
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND)
#endif
# 299 "beam/sys.h"

#if SIZEOF_VOID_P != SIZEOF_SIZE_T
#error sizeof(void*) != sizeof(size_t)
#endif
# 303 "beam/sys.h"

#if HALFWORD_HEAP

#if SIZEOF_INT == 4
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define ERTS_UINT_MAX UINT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#else
# 314 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 316 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long UWord;
typedef long          SWord;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#elif SIZEOF_VOID_P == SIZEOF_INT
# 325 "beam/sys.h"
typedef unsigned int UWord;
typedef int          SWord;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 332 "beam/sys.h"
typedef unsigned long long UWord;
typedef long long          SWord;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#else
# 339 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 341 "beam/sys.h"

#else /* !HALFWORD_HEAP */
# 343 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long Eterm;
typedef unsigned long Uint;
typedef long          Sint;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_INT
# 355 "beam/sys.h"
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 365 "beam/sys.h"
typedef unsigned long long Eterm;
typedef unsigned long long Uint;
typedef long long          Sint;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
#if defined(__WIN32__)
#define ErtsStrToSint _strtoi64
#else
# 376 "beam/sys.h"
#define ErtsStrToSint strtoll
#endif
# 378 "beam/sys.h"
#else
# 379 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 381 "beam/sys.h"

typedef Uint UWord;
typedef Sint SWord;
#define ERTS_UINT_MAX ERTS_UWORD_MAX

#endif /* HALFWORD_HEAP */
# 387 "beam/sys.h"

typedef UWord BeamInstr;

#ifndef HAVE_INT64
#  if SIZEOF_LONG == 8
#    define HAVE_INT64 1
typedef unsigned long Uint64;
typedef long          Sint64;
#    ifdef ULONG_MAX
#      define ERTS_UINT64_MAX ULONG_MAX
#    endif
# 398 "beam/sys.h"
#    ifdef LONG_MAX
#      define ERTS_SINT64_MAX LONG_MAX
#    endif
# 401 "beam/sys.h"
#    ifdef LONG_MIN
#      define ERTS_SINT64_MIN LONG_MIN
#    endif
# 404 "beam/sys.h"
#  elif SIZEOF_LONG_LONG == 8
# 405 "beam/sys.h"
#    define HAVE_INT64 1
typedef unsigned long long Uint64;
typedef long long          Sint64;
#    ifdef ULLONG_MAX
#      define ERTS_UINT64_MAX ULLONG_MAX
#    endif
# 411 "beam/sys.h"
#    ifdef LLONG_MAX
#      define ERTS_SINT64_MAX LLONG_MAX
#    endif
# 414 "beam/sys.h"
#    ifdef LLONG_MIN
#      define ERTS_SINT64_MIN LLONG_MIN
#    endif
# 417 "beam/sys.h"
#  else
# 418 "beam/sys.h"
#    error "No 64-bit integer type found"
#  endif
# 420 "beam/sys.h"
#endif
# 421 "beam/sys.h"

#ifndef ERTS_UINT64_MAX
#  define ERTS_UINT64_MAX (~((Uint64) 0))
#endif
# 425 "beam/sys.h"
#ifndef ERTS_SINT64_MAX
#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
#endif
# 428 "beam/sys.h"
#ifndef ERTS_SINT64_MIN
#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
#endif
# 431 "beam/sys.h"

#if SIZEOF_LONG == 4
typedef unsigned long Uint32;
typedef long          Sint32;
#elif SIZEOF_INT == 4
# 436 "beam/sys.h"
typedef unsigned int Uint32;
typedef int          Sint32;
#else
# 439 "beam/sys.h"
#error Found no appropriate type to use for 'Uint32' and 'Sint32'
#endif
# 441 "beam/sys.h"

#if SIZEOF_INT == 2
typedef unsigned int Uint16;
typedef int          Sint16;
#elif SIZEOF_SHORT == 2
# 446 "beam/sys.h"
typedef unsigned short Uint16;
typedef short          Sint16;
#else
# 449 "beam/sys.h"
#error Found no appropriate type to use for 'Uint16' and 'Sint16'
#endif
# 451 "beam/sys.h"

#if CHAR_BIT == 8
typedef unsigned char byte;
#else
# 455 "beam/sys.h"
#error Found no appropriate type to use for 'byte'
#endif
# 457 "beam/sys.h"

#if defined(ARCH_64) && !HAVE_INT64
#error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
#endif
# 461 "beam/sys.h"

#ifdef WORDS_BIGENDIAN
#  define ERTS_HUINT_HVAL_HIGH 0
#  define ERTS_HUINT_HVAL_LOW 1
#else
# 466 "beam/sys.h"
#  define ERTS_HUINT_HVAL_HIGH 1
#  define ERTS_HUINT_HVAL_LOW 0
#endif
# 469 "beam/sys.h"
#if ERTS_SIZEOF_TERM == 8
typedef union {
    Uint val;
    Uint32 hval[2];
} HUint;
#elif ERTS_SIZEOF_TERM == 4
# 475 "beam/sys.h"
typedef union {
    Uint val;
    Uint16 hval[2];
} HUint;
#else
# 480 "beam/sys.h"
#error "Unsupported size of term"
#endif
# 482 "beam/sys.h"

#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
    (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 486 "beam/sys.h"
# 487 "beam/sys.h"

/* needed by erl_smp.h */
int erts_send_warning_to_logger_str_nogl(char *);

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/sys.h"
# 492 "beam/sys.h"

#ifdef ERTS_WANT_BREAK_HANDLING
#  ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#    define ERTS_BREAK_REQUESTED \
  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
#  else
# 499 "beam/sys.h"
extern volatile int erts_break_requested;
#    define ERTS_BREAK_REQUESTED erts_break_requested
#  endif
# 502 "beam/sys.h"
void erts_do_break_handling(void);
#endif
# 504 "beam/sys.h"

#ifdef ERTS_WANT_GOT_SIGUSR1
#  ifndef UNIX
#    define ERTS_GOT_SIGUSR1 0
#  else
# 509 "beam/sys.h"
#    ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
#    else
# 513 "beam/sys.h"
extern volatile int erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
#    endif
# 516 "beam/sys.h"
#  endif
# 517 "beam/sys.h"
#endif
# 518 "beam/sys.h"

#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
extern erts_tsd_key_t erts_is_crash_dumping_key;
#define ERTS_SOMEONE_IS_CRASH_DUMPING \
  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
# 527 "beam/sys.h"
extern volatile int erts_writing_erl_crash_dump;
#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
# 531 "beam/sys.h"

/* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
   but be able to fall back on bcopy() etc on systems that don't have
   mem*(), but this doesn't work to well with memset()/bzero() - thus the
   memzero() macro.
*/

/* xxxP */
#if defined(USE_BCOPY)
#  define memcpy(a, b, c) bcopy((b), (a), (c))
#  define memcmp(a, b, c) bcmp((a), (b), (c))
#  define memzero(buf, len) bzero((buf), (len))
#else
# 544 "beam/sys.h"
#  define memzero(buf, len) memset((buf), '\0', (len))
#endif
# 546 "beam/sys.h"

/* Stuff that is useful for port programs, drivers, etc */

#ifdef ISC32			/* Too much for the Makefile... */
#  define signal	sigset
#  define NO_ASINH
#  define NO_ACOSH
#  define NO_ATANH
#  define NO_FTRUNCATE
#  define SIG_SIGHOLD
#  define _POSIX_SOURCE 
#  define _XOPEN_SOURCE
#endif
# 559 "beam/sys.h"

#ifdef QNX			/* Too much for the Makefile... */
#  define SYS_SELECT_H
#  define NO_ERF
#  define NO_ERFC
/* This definition doesn't take NaN into account, but matherr() gets those */
#  define isfinite(x) (fabs(x) != HUGE_VAL)
#  define USE_MATHERR
#  define HAVE_FINITE
#endif
# 569 "beam/sys.h"


#ifdef WANT_NONBLOCKING	    /* must define this to pull in fcntl.h/ioctl.h */

/* This is really a mess... We used to use fcntl O_NDELAY, but that seems
   to only work on SunOS 4 - in particular, on SysV-based systems
   (including Solaris 2), it does set non-blocking mode, but causes
   read() to return 0!!  fcntl O_NONBLOCK is specified by POSIX, and
   seems to work on most systems, with the notable exception of AIX,
   where the old ioctl FIONBIO is the *only* one that will set a *socket*
   in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
   pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */

#  ifdef __WIN32__

static unsigned long zero_value = 0, one_value = 1;
#    define SET_BLOCKING(fd)	{ if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); }
#    define SET_NONBLOCKING(fd)	ioctlsocket((fd), FIONBIO, &one_value)

#  else
# 589 "beam/sys.h"
#    ifdef NB_FIONBIO		/* Old BSD */
#if 0 /* expanded by -frewrite-includes */
#      include <sys/ioctl.h>
#endif /* expanded by -frewrite-includes */
# 590 "beam/sys.h"
# 591 "beam/sys.h"
  static const int zero_value = 0, one_value = 1;
#      define SET_BLOCKING(fd)         ioctl((fd), FIONBIO, &zero_value)
#      define SET_NONBLOCKING(fd)      ioctl((fd), FIONBIO, &one_value)
#      define ERRNO_BLOCK EWOULDBLOCK
#    else /* !NB_FIONBIO */
# 596 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#      include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 596 "beam/sys.h"
# 597 "beam/sys.h"
#      ifdef NB_O_NDELAY               /* Nothing needs this? */
#        define NB_FLAG O_NDELAY
#        ifndef ERRNO_BLOCK            /* allow override (e.g. EAGAIN) via Makefile */
#          define ERRNO_BLOCK EWOULDBLOCK
#        endif
# 602 "beam/sys.h"
#      else  /* !NB_O_NDELAY */	/* The True Way - POSIX!:-) */
# 603 "beam/sys.h"
#        define NB_FLAG O_NONBLOCK
#        define ERRNO_BLOCK EAGAIN
#      endif /* !NB_O_NDELAY */
# 606 "beam/sys.h"
#      define SET_BLOCKING(fd)         fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) & ~NB_FLAG)
#      define SET_NONBLOCKING(fd)      fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) | NB_FLAG)
#    endif /* !NB_FIONBIO */
# 611 "beam/sys.h"
#  endif /* !__WIN32__ */
# 612 "beam/sys.h"
#endif /* WANT_NONBLOCKING */
# 613 "beam/sys.h"

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);

/* Some special erts_exit() codes: */
#define ERTS_INTR_EXIT	-1		/* called from signal handler */
#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */

#define ERTS_INTERNAL_ERROR(What) \
    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
	     __FILE__, __LINE__, __func__, What)

Eterm erts_check_io_info(void *p);

/* Size of misc memory allocated from system dependent code */
Uint erts_sys_misc_mem_sz(void);

/* print stuff is declared here instead of in global.h, so sys stuff won't
   have to include global.h */
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 633 "beam/sys.h"
# 634 "beam/sys.h"

/* Io constants to erts_print and erts_putc */
#define ERTS_PRINT_STDERR	(2)
#define ERTS_PRINT_STDOUT	(1)
#define ERTS_PRINT_FILE		(-1)
#define ERTS_PRINT_SBUF		(-2)
#define ERTS_PRINT_SNBUF	(-3)
#define ERTS_PRINT_DSBUF	(-4)

#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF

typedef struct {
    char *buf;
    size_t size;
} erts_print_sn_buf;

int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
int erts_putc(int to, void *arg, char);			/* in utils.c */

/* logger stuff is declared here instead of in global.h, so sys files
   won't have to include global.h */

erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
int erts_send_info_to_logger_str(Eterm, char *); 
int erts_send_warning_to_logger_str(Eterm, char *);
int erts_send_error_to_logger_str(Eterm, char *);
int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
/* needed by erl_smp.h (declared above)
   int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);

typedef struct preload {
    char *name;			/* Name of module */
    int  size;			/* Size of code */
    unsigned char* code;	/* Code pointer */
} Preload;


/*
 * This structure contains options to all built in drivers.
 * None of the drivers use all of the fields.
 */

typedef struct _SysDriverOpts {
    Uint ifd;			/* Input file descriptor (fd driver). */
    Uint ofd;			/* Outputfile descriptor (fd driver). */
    int packet_bytes;		/* Number of bytes in packet header. */
    int read_write;		/* Read and write bits. */
    int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
    int redir_stderr;           /* Redirect stderr to stdout: TRUE/FALSE. */
    int hide_window;		/* Hide this windows (Windows). */
    int exit_status;		/* Report exit status of subprocess. */
    int overlapped_io;          /* Only has effect on windows NT et al */
    char *envir;		/* Environment of the port process, */
				/* in Windows format. */
    char **argv;                /* Argument vector in Unix'ish format. */
    char *wd;			/* Working directory. */
    unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
				   ERTS_SPAWN_EXTERNAL | both*/ 
    int parallelism;            /* Optimize for parallelism */
} SysDriverOpts;

extern char *erts_default_arg0;

extern char os_type[];

typedef struct {
    int have_os_monotonic_time;
    int have_corrected_os_monotonic_time;
    ErtsMonotonicTime os_monotonic_time_unit;
    ErtsMonotonicTime sys_clock_resolution;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
	int extended;
    } os_monotonic_time_info;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
    } os_system_time_info;
} ErtsSysInitTimeResult;

#define ERTS_SYS_INIT_TIME_RESULT_INITER \
    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}

extern void erts_init_sys_time_sup(void);
extern void sys_init_time(ErtsSysInitTimeResult *);
extern void erts_late_sys_init_time(void);
extern void erts_deliver_time(void);
extern void erts_time_remaining(SysTimeval *);
extern void erts_sys_init_float(void);
extern void erts_thread_init_float(void);
extern void erts_thread_disable_fpe(void);
ERTS_GLB_INLINE int erts_block_fpe(void);
ERTS_GLB_INLINE void erts_unblock_fpe(int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int erts_block_fpe(void)
{
    return erts_sys_block_fpe();
}

ERTS_GLB_INLINE void erts_unblock_fpe(int unmasked)
{
    erts_sys_unblock_fpe(unmasked);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 754 "beam/sys.h"


/* Dynamic library/driver loading */
typedef struct {
    char* str;
}ErtsSysDdllError;
#define ERTS_SYS_DDLL_ERROR_INIT {NULL}
extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*);
#define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL)
extern void *erts_sys_ddll_call_init(void *function);
extern void *erts_sys_ddll_call_nif_init(void *function);
extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*);
#define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL)
extern char *erts_sys_ddll_error(int code);



/*
 * System interfaces for startup.
 */
void erts_sys_schedule_interrupt(int set);
#ifdef ERTS_SMP
void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
void erts_sys_main_thread(void);
#endif
# 785 "beam/sys.h"

extern int erts_sys_prepare_crash_dump(int secs);
extern void erts_sys_pre_init(void);
extern void erl_sys_init(void);
extern void erl_sys_args(int *argc, char **argv);
extern void erl_sys_schedule(int);
void sys_tty_reset(int);

int sys_max_files(void);
void sys_init_io(void);
Preload* sys_preloaded(void);
unsigned char* sys_preload_begin(Preload*);
void sys_preload_end(Preload*);
int sys_get_key(int);
void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
		       UWord *ms_user_diff, UWord *ms_sys_diff);
void wall_clock_elapsed_time_both(UWord *ms_total, 
				  UWord *ms_diff);
void get_time(int *hour, int *minute, int *second);
void get_date(int *year, int *month, int *day);
void get_localtime(int *year, int *month, int *day, 
		   int *hour, int *minute, int *second);
void get_universaltime(int *year, int *month, int *day, 
		       int *hour, int *minute, int *second);
int seconds_to_univ(Sint64 seconds, 
		    Sint *year, Sint *month, Sint *day, 
		    Sint *hour, Sint *minute, Sint *second);
int univ_to_seconds(Sint year, Sint month, Sint day, 
		    Sint hour, Sint minute, Sint second,
		    Sint64* seconds);
int univ_to_local(
    Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second);
int local_to_univ(Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second, int isdst);
void get_now(Uint*, Uint*, Uint*);
struct ErtsSchedulerData_;
ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
ErtsMonotonicTime erts_get_time_offset(void);
void
erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
void get_sys_now(Uint*, Uint*, Uint*);
void set_break_quit(void (*)(void), void (*)(void));

void os_flavor(char*, unsigned);
void os_version(int*, int*, int*);
void init_getenv_state(GETENV_STATE *);
char * getenv_string(GETENV_STATE *);
void fini_getenv_state(GETENV_STATE *);

#define HAVE_ERTS_CHECK_IO_DEBUG
typedef struct {
    int no_used_fds;
    int no_driver_select_structs;
    int no_driver_event_structs;
} ErtsCheckIoDebugInfo;
int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);

int erts_sys_is_area_readable(char *start, char *stop);

/* xxxP */
#define SYS_DEFAULT_FLOAT_DECIMALS 20
void init_sys_float(void);
int sys_chars_to_double(char*, double*);
int sys_double_to_chars(double, char*, size_t);
int sys_double_to_chars_ext(double, char*, size_t, size_t);
int sys_double_to_chars_fast(double, char*, int, int, int);
void sys_get_pid(char *, size_t);

/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
int erts_sys_putenv(char *key, char *value);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_putenv_raw(char *key, char *value);
/* erts_sys_getenv() returns 0 on success (length of value string in
   *size), a value > 0 if value buffer is too small (*size is set to needed
   size), and a value < 0 on failure. */
int erts_sys_getenv(char *key, char *value, size_t *size);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_getenv_raw(char *key, char *value, size_t *size);
/* erts_sys_getenv__() is only allowed to be used in early init phase */
int erts_sys_getenv__(char *key, char *value, size_t *size);
/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
int erts_sys_unsetenv(char *key);

/* Easier to use, but not as efficient, environment functions */
char *erts_read_env(char *key);
void erts_free_read_env(void *value);

#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
extern void sys_thr_resume(erts_tid_t tid);
extern void sys_thr_suspend(erts_tid_t tid);
#endif
# 878 "beam/sys.h"

/* utils.c */

/* Options to sys_alloc_opt */
#define SYS_ALLOC_OPT_TRIM_THRESHOLD 0
#define SYS_ALLOC_OPT_TOP_PAD        1
#define SYS_ALLOC_OPT_MMAP_THRESHOLD 2
#define SYS_ALLOC_OPT_MMAP_MAX       3

/* Default values to sys_alloc_opt options */
#define ERTS_DEFAULT_TRIM_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_TOP_PAD         0
#define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_MMAP_MAX        64

int sys_alloc_opt(int, int);

typedef struct {
  int trim_threshold;
  int top_pad;
  int mmap_threshold;
  int mmap_max;
} SysAllocStat;

void sys_alloc_stat(SysAllocStat *);

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#undef ERTS_REFC_DEBUG
#define ERTS_REFC_DEBUG
#endif
# 908 "beam/sys.h"

typedef erts_smp_atomic_t erts_refc_t;

ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
				   erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
					   erts_aint_t min_val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}

ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 941 "beam/sys.h"
    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
# 943 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 955 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 968 "beam/sys.h"
    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
# 970 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 982 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
		 diff, val, min_val);
#else
# 995 "beam/sys.h"
    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
# 997 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 1009 "beam/sys.h"
    return val;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1013 "beam/sys.h"

#ifdef ERTS_ENABLE_KERNEL_POLL
extern int erts_use_kernel_poll;
#endif
# 1017 "beam/sys.h"

#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
#define sys_memset(s,c,n)    memset(s,c,n)
#define sys_memzero(s, n)    memset(s,'\0',n)
#define sys_strcmp(s1,s2)    strcmp(s1,s2)
#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
#define sys_strcpy(s1,s2)    strcpy(s1,s2)
#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
#define sys_strlen(s)        strlen(s)

/* define function symbols (needed in sys_drv_api) */
#define sys_fp_alloc     sys_alloc
#define sys_fp_realloc   sys_realloc
#define sys_fp_free      sys_free
#define sys_fp_memcpy    memcpy
#define sys_fp_memmove   memmove
#define sys_fp_memcmp    memcmp
#define sys_fp_memset    memset
/* #define sys_fp_memzero    elib_memzero */
#define sys_fp_strcmp    strcmp
#define sys_fp_strncmp   strncmp
#define sys_fp_strcpy    strcpy
#define sys_fp_strncpy   strncpy
#define sys_fp_strlen    strlen


/* Return codes from the nb_read and nb_write functions */
#define FD_READY 1
#define FD_CONTINUE 2
#define FD_ERROR 3



/* Standard set of integer macros  .. */

#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
                      (((Uint64)((unsigned char*) (s))[7])))

#define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                            ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
                            ((char*)(s))[2] = (char)((Sint64)(i) >> 40) & 0xff;\
                            ((char*)(s))[3] = (char)((Sint64)(i) >> 32) & 0xff;\
                            ((char*)(s))[4] = (char)((Sint64)(i) >> 24) & 0xff;\
                            ((char*)(s))[5] = (char)((Sint64)(i) >> 16) & 0xff;\
                            ((char*)(s))[6] = (char)((Sint64)(i) >> 8)  & 0xff;\
                            ((char*)(s))[7] = (char)((Sint64)(i))       & 0xff;\
                           } while (0) 

#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \
                      (((unsigned char*) (s))[1] << 16) | \
                      (((unsigned char*) (s))[2] << 8)  | \
                      (((unsigned char*) (s))[3]))

#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff;   \
                            ((char*)(s))[1] = (char)((i) >> 16) & 0xff;   \
                            ((char*)(s))[2] = (char)((i) >> 8)  & 0xff;   \
                            ((char*)(s))[3] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
                      (((unsigned char*) (s))[1] << 8)  | \
                      (((unsigned char*) (s))[2]))

#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                      (((unsigned char*)  (s))[1]))


#define put_int16(i, s) do {((char*)(s))[0] = (char)((i) >> 8) & 0xff;  \
                            ((char*)(s))[1] = (char)(i)        & 0xff;} \
                        while (0)

#define get_int8(s) ((((unsigned char*)  (s))[0] ))


#define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)


/*
 * Use DEBUGF as you would use printf, but use double parentheses:
 *
 *   DEBUGF(("Error: %s\n", error));
 *
 * The output will appear in a special console.
 */

#ifdef DEBUG
void erl_debug(char* format, ...);
void erl_bin_write(unsigned char *, int, int);

#  define DEBUGF(x) erl_debug x
#else
# 1121 "beam/sys.h"
#  define DEBUGF(x)
#endif
# 1123 "beam/sys.h"

#ifdef __WIN32__
#ifdef ARCH_64
#define ERTS_ALLOC_ALIGN_BYTES 16
#define ERTS_SMALL_ABS(Small) _abs64(Small) 
#else
# 1129 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1132 "beam/sys.h"
#else
# 1133 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1136 "beam/sys.h"

#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
#endif
# 1140 "beam/sys.h"

#ifdef __WIN32__
void call_break_handler(void);
char* last_error(void);
char* win32_errorstr(int);
#endif
# 1146 "beam/sys.h"

/************************************************************************
 * Find out the native filename encoding of the process (look at locale of 
 * Unix processes and just do UTF16 on windows 
 ************************************************************************/
#define ERL_FILENAME_UNKNOWN   (0)
#define ERL_FILENAME_LATIN1    (1)
#define ERL_FILENAME_UTF8      (2)
#define ERL_FILENAME_UTF8_MAC  (3)
#define ERL_FILENAME_WIN_WCHAR (4)

/************************************************************************
 * If a filename in for example list_dir is not in the right encoding, it
 * will be skipped in the resulting list, but depending on a startup setting
 * we will inform the user in different ways. These macros define the
 * different reactions to wrongly coded filenames. In the error case an
 * exception will be thrown by prim_file.
 ************************************************************************/
#define ERL_FILENAME_WARNING_WARNING (0)
#define ERL_FILENAME_WARNING_IGNORE (1)
#define ERL_FILENAME_WARNING_ERROR (2)

/***********************************************************************
 * The user can request a range of character that he/she consider
 * printable. Currently this can be either latin1 or unicode, but
 * in the future a set of ranges, or languages, could be specified.
 ***********************************************************************/
#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0)
#define ERL_PRINTABLE_CHARACTERS_UNICODE (1)

int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
void erts_set_user_requested_filename_encoding(int encoding, int warning);
int erts_get_user_requested_filename_encoding(void);
int erts_get_filename_warning_type(void);
/* This function is called from erl_init. The setting is read by BIF's 
   in io/io_lib. Setting is not atomic. */
void erts_set_printable_characters(int range);
/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */
int erts_get_printable_characters(void);

void erts_init_sys_common_misc(void);

#endif
# 1190 "beam/sys.h"
# 33 "beam/erl_lock_check.h" 2

#ifndef ERTS_LOCK_CHECK_H__
#define ERTS_LOCK_CHECK_H__

#ifdef ERTS_ENABLE_LOCK_CHECK

#ifndef ERTS_ENABLE_LOCK_POSITION
/* Enable in order for _x variants of mtx functions to be used. */
#define ERTS_ENABLE_LOCK_POSITION 1
#endif
# 43 "beam/erl_lock_check.h"

typedef struct {
    int inited;
    Sint16 id;
    Uint16 flags;
    UWord extra;
} erts_lc_lock_t;

#define ERTS_LC_INITITALIZED 0x7f7f7f7f


#define ERTS_LC_FLG_LT_SPINLOCK		(((Uint16) 1) << 0)
#define ERTS_LC_FLG_LT_RWSPINLOCK	(((Uint16) 1) << 1)
#define ERTS_LC_FLG_LT_MUTEX		(((Uint16) 1) << 2)
#define ERTS_LC_FLG_LT_RWMUTEX		(((Uint16) 1) << 3)
#define ERTS_LC_FLG_LT_PROCLOCK		(((Uint16) 1) << 4)

#define ERTS_LC_FLG_LO_READ		(((Uint16) 1) << 5)
#define ERTS_LC_FLG_LO_WRITE		(((Uint16) 1) << 6)

#define ERTS_LC_FLG_LO_READ_WRITE	(ERTS_LC_FLG_LO_READ		\
					 | ERTS_LC_FLG_LO_WRITE)

#define ERTS_LC_FLG_LT_ALL		(ERTS_LC_FLG_LT_SPINLOCK	\
					 | ERTS_LC_FLG_LT_RWSPINLOCK	\
					 | ERTS_LC_FLG_LT_MUTEX		\
					 | ERTS_LC_FLG_LT_RWMUTEX	\
					 | ERTS_LC_FLG_LT_PROCLOCK)

#define ERTS_LC_FLG_LO_ALL		(ERTS_LC_FLG_LO_READ		\
					 | ERTS_LC_FLG_LO_WRITE)


#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), (X)}

void erts_lc_init(void);
void erts_lc_late_init(void);
Sint16 erts_lc_get_lock_order_id(char *name);
void erts_lc_check(erts_lc_lock_t *have, int have_len,
		   erts_lc_lock_t *have_not, int have_not_len);
void erts_lc_check_exact(erts_lc_lock_t *have, int have_len);
void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len);
void erts_lc_have_lock_ids(int *resv, int *ids, int len);
void erts_lc_check_no_locked_of_type(Uint16 flags);
int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags);
void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
			   char *file, unsigned int line);
void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
			char *file, unsigned int line);
void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
int erts_lc_trylock_force_busy(erts_lc_lock_t *lck);
void erts_lc_trylock_x(int locked, erts_lc_lock_t *lck,
		     char* file, unsigned int line);
void erts_lc_lock_x(erts_lc_lock_t *lck, char* file, unsigned int line);
void erts_lc_unlock(erts_lc_lock_t *lck);
void erts_lc_might_unlock(erts_lc_lock_t *lck);
void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags);
void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra);
void erts_lc_destroy_lock(erts_lc_lock_t *lck);
void erts_lc_fail(char *fmt, ...);
int erts_lc_assert_failed(char *file, int line, char *assertion);
void erts_lc_set_thread_name(char *thread_name);
void erts_lc_pll(void);

void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
			      char *file, unsigned int line);
void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);

void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line);
void erts_lc_unrequire_lock(erts_lc_lock_t *lck);

int erts_lc_is_emu_thr(void);

#define ERTS_LC_ASSERT(A) \
    ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A)))
#ifdef ERTS_SMP
#define ERTS_SMP_LC_ASSERT(A) ERTS_LC_ASSERT(A)
#else
# 122 "beam/erl_lock_check.h"
#define ERTS_SMP_LC_ASSERT(A) ((void) 1)
#endif
# 124 "beam/erl_lock_check.h"
#else /* #ifdef ERTS_ENABLE_LOCK_CHECK */
# 125 "beam/erl_lock_check.h"
#define ERTS_SMP_LC_ASSERT(A) ((void) 1)
#define ERTS_LC_ASSERT(A) ((void) 1)
#endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
# 128 "beam/erl_lock_check.h"

#define erts_lc_lock(lck) erts_lc_lock_x(lck,__FILE__,__LINE__)
#define erts_lc_trylock(res,lck) erts_lc_trylock_x(res,lck,__FILE__,__LINE__)
#define erts_lc_lock_flg(lck) erts_lc_lock_flg_x(lck,__FILE__,__LINE__)
#define erts_lc_trylock_flg(res,lck) erts_lc_trylock_flg_x(res,lck,__FILE__,__LINE__)

#endif /* #ifndef ERTS_LOCK_CHECK_H__ */
# 135 "beam/erl_lock_check.h"
# 487 "beam/sys.h" 2

/* needed by erl_smp.h */
int erts_send_warning_to_logger_str_nogl(char *);

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/sys.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 1 "beam/erl_threads.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Description: Error checking thread interface to the ethread library.
 *              All functions terminates the emulator on failure.
 * Author: Rickard Green
 */

#ifndef ERL_THREAD_H__
#define ERL_THREAD_H__

/*
 * --- Documentation of atomics and memory barriers --------------------------
 *
 * The following explicit memory barriers exist:
 *   
 * - ERTS_THR_MEMORY_BARRIER
 *      Full memory barrier. Orders both loads, and stores. No
 *      load or store is allowed to be reordered over the
 *      barrier.
 * - ERTS_THR_WRITE_MEMORY_BARRIER
 *      Write barrier. Orders *only* stores. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_READ_MEMORY_BARRIER
 *      Read barrier. Orders *only* loads. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 *      Data dependency read barrier. Orders *only* loads
 *      according to data dependency across the barrier.
 *
 * If thread support has been disabled, these barriers will become no-ops.
 *
 * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
 * be enabled only in the SMP enabled runtime system.
 *
 * --- Atomic operations ---
 *
 * Atomics operations exist for 32-bit, word size, and double word size
 * integers. Function prototypes are listed below.
 *
 * Each function implementing an atomic operation exist with the following
 * implied memory barrier semantics. Not all combinations are useful, but
 * all of them exist for simplicity. <B> is suffix in function name:
 *
 * - <B>  - Description
 *
 * - mb   - Full memory barrier. Orders both loads, and
 *          stores before, and after the atomic operation.
 *          No load or store is allowed to be reordered
 *          over the atomic operation.
 * - relb - Release barrier. Orders both loads, and
 *          stores appearing *before* the atomic
 *          operation. These are not allowed to be
 *          reordered over the atomic operation.
 * - acqb - Acquire barrier. Orders both loads, and stores
 *          appearing *after* the atomic operation. These
 *          are not allowed to be reordered over the
 *          atomic operation.
 * - wb   - Write barrier. Orders *only* stores. These are
 *          not allowed to be reordered over the barrier.
 *          Store in atomic operation is ordered *after*
 *          the barrier.
 * - rb   - Read barrier. Orders *only* loads. These are
 *          not allowed to be reordered over the barrier.
 *          Load in atomic operation is ordered *before*
 *          the barrier. 
 * - ddrb - Data dependency read barrier. Orders *only*
 *          loads according to data dependency across the
 *          barrier. Load in atomic operation is ordered
 *          before the barrier.
 *
 * If thread support has been disabled, these functions are mapped to
 * functions that performs the same operation, but aren't atomic
 * and don't imply any memory barriers.
 *
 * If the atomic operations are prefixed with erts_smp_ instead of only
 * erts_ the atomic operations will only be atomic in the SMP enabled
 * runtime system, and will be mapped to non-atomic operations without
 * memory barriers in the runtime system without SMP support. Atomic
 * operations with erts_smp_ prefix should use the atomic types
 * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
 * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
 * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
 * are the same.
 *
 * --- 32-bit atomic operations ---
 *
 * The following 32-bit atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above). Note
 * that sizeof(erts_atomic32_t) might be larger than 4!
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
 *                               erts_aint32_t val);
 *
 * Set value:
 *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Read; returns current value:
 *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment:
 *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
 *
 * Decrement:
 *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Add value:
 *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
 *                                             erts_aint32_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
 *                                        erts_aint32_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
 *                                           erts_aint32_t new,
 *                                           erts_aint32_t exp);
 *
 * --- Word size atomic operations ---
 *
 * The following word size (same size as sizeof(void *)) atomic
 * operations exist. <B> should be replaced with a supported
 * memory barrier (see above). Note that sizeof(erts_atomic_t)
 * might be larger than sizeof(void *)!
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
 *                             erts_aint_t val);
 *
 * Set value;
 *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Read; returns current value:
 *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
 *
 * Increment:
 *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
 *
 * Decrement:
 *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Add value:
 *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
 *                                         erts_aint_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
 *                                    erts_aint_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
 *                                       erts_aint_t new,
 *                                       erts_aint_t exp);
 *
 * --- Double word size atomic operations ---
 *
 * The following double word atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above).
 *
 * Note that sizeof(erts_dw_atomic_t) usually is larger than
 * 2*sizeof(void *)!
 * 
 * The erts_dw_aint_t data type should be accessed as if it was defined
 * like this:
 *
 *     typedef struct {
 *         erts_aint_t sint[2];
 *     } erts_dw_aint_t;
 *
 *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
 *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
 *                           erts_dw_aint_t *val);
 *
 * Set; value is written into 'val':
 *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
 *                               erts_dw_aint_t *val);
 *
 * Read; value is written into 'val':
 *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
 *                                erts_dw_aint_t *val);
 *
 * Compare and exchange; returns a value != 0 if exchange was
 * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
 * contains the same value as in memory when the function is called,
 * 'new' is written to memory; otherwise, not. If exchange was not
 * made, 'exp_act' contains the actual value in memory:
 *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
 *                                  erts_dw_aint_t *new,
 *                                  erts_dw_aint_t *exp_act);
 */

#define ERTS_SPIN_BODY ETHR_SPIN_BODY

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 260 "beam/erl_threads.h"
# 1 "beam/sys.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __SYS_H__
#define __SYS_H__

#ifdef ERTS_INLINE
#  ifndef ERTS_CAN_INLINE
#    define ERTS_CAN_INLINE 1
#  endif
# 28 "beam/sys.h"
#else
# 29 "beam/sys.h"
#  if defined(__GNUC__)
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline__
#  elif defined(__WIN32__)
# 33 "beam/sys.h"
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline
#  else
# 36 "beam/sys.h"
#    define ERTS_CAN_INLINE 0
#    define ERTS_INLINE
#  endif
# 39 "beam/sys.h"
#endif
# 40 "beam/sys.h"

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#  undef ERTS_CAN_INLINE
#  define ERTS_CAN_INLINE 0
#  undef ERTS_INLINE
#  define ERTS_INLINE
#endif
# 47 "beam/sys.h"

#if ERTS_CAN_INLINE
#define ERTS_GLB_INLINE static ERTS_INLINE
#else
# 51 "beam/sys.h"
#define ERTS_GLB_INLINE
#endif
# 53 "beam/sys.h"

#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
#else
# 57 "beam/sys.h"
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
# 59 "beam/sys.h"

#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
#  define NO_FPE_SIGNALS
#endif
# 63 "beam/sys.h"

#ifdef DISABLE_CHILD_WAITER_THREAD
#undef ENABLE_CHILD_WAITER_THREAD
#endif
# 67 "beam/sys.h"

#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
#undef ENABLE_CHILD_WAITER_THREAD
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
# 72 "beam/sys.h"

#define ERTS_I64_LITERAL(X) X##LL

#if defined (__WIN32__)
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_sys.h"
#endif /* expanded by -frewrite-includes */
# 76 "beam/sys.h"
# 77 "beam/sys.h"
#elif defined (__OSE__)
# 78 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_ose_sys.h"
#endif /* expanded by -frewrite-includes */
# 78 "beam/sys.h"
# 79 "beam/sys.h"
#else 
# 80 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_unix_sys.h"
#endif /* expanded by -frewrite-includes */
# 80 "beam/sys.h"
# 81 "beam/sys.h"
#ifndef UNIX
#  define UNIX 1
#endif
# 84 "beam/sys.h"
#endif
# 85 "beam/sys.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_misc_utils.h"
#endif /* expanded by -frewrite-includes */
# 86 "beam/sys.h"
# 87 "beam/sys.h"

/*
 * To allow building of Universal Binaries for Mac OS X,
 * we must not depend on the endian detected by the configure script.
 */
#if defined(__APPLE__)
#  if defined(__BIG_ENDIAN__) && !defined(WORDS_BIGENDIAN)
#    define WORDS_BIGENDIAN 1
#  elif !defined(__BIG_ENDIAN__) && defined(WORDS_BIGENDIAN)
# 96 "beam/sys.h"
#    undef WORDS_BIGENDIAN
#  endif
# 98 "beam/sys.h"
#endif
# 99 "beam/sys.h"

/*
 * Make sure we have a type for FD's (used by erl_check_io)
 */

#ifndef ERTS_SYS_FD_TYPE
#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
typedef int ErtsSysFdType;
#else
# 108 "beam/sys.h"
#ifndef ERTS_SYS_FD_INVALID
# error missing ERTS_SYS_FD_INVALID
#endif
# 111 "beam/sys.h"
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
# 113 "beam/sys.h"

#if !defined(__GNUC__)
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 117 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 120 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 123 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 126 "beam/sys.h"

#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
#  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
#  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# 131 "beam/sys.h"
#  define ERTS_LIKELY(BOOL)   (BOOL)
#  define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
# 134 "beam/sys.h"
#ifdef __GNUC__
#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
#  else
# 138 "beam/sys.h"
#    define ERTS_DECLARE_DUMMY(X) X
#  endif
# 140 "beam/sys.h"
#else
# 141 "beam/sys.h"
#  define ERTS_DECLARE_DUMMY(X) X
#endif
# 143 "beam/sys.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 149 "beam/sys.h"
#      define __func__ __FUNCTION__
#    endif
# 151 "beam/sys.h"
#  endif
# 152 "beam/sys.h"
#endif
# 153 "beam/sys.h"

#define ERTS_MK_VSN_INT(Major, Minor, Build) \
    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))

#ifndef ERTS_EXIT_AFTER_DUMP
#  define ERTS_EXIT_AFTER_DUMP exit
#endif
# 160 "beam/sys.h"

/* In VC++, noreturn is a declspec that has to be before the types,
 * but in GNUC it is an att ribute to be placed between return type
 * and function name, hence __decl_noreturn <types> __noreturn <function name>
 *
 * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
 */
#if __GNUC__
#  define __decl_noreturn
#  ifndef __noreturn
#     define __noreturn __attribute__((noreturn))
#  endif
# 172 "beam/sys.h"
#else
# 173 "beam/sys.h"
#  if defined(__WIN32__) && defined(_MSC_VER)
#    define __noreturn
#    define __decl_noreturn __declspec(noreturn)
#  else
# 177 "beam/sys.h"
#    define __noreturn
#    define __decl_noreturn
#  endif
# 180 "beam/sys.h"
#endif
# 181 "beam/sys.h"

#define ERTS_ASSERT(e) \
    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))

__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
						 const char* file, int line);

#ifdef DEBUG
#  define ASSERT(e) ERTS_ASSERT(e)
#else
# 191 "beam/sys.h"
#  define ASSERT(e) ((void) 1)
#endif
# 193 "beam/sys.h"

/* ERTS_UNDEF can be used to silence false warnings about
 * "variable may be used uninitialized" while keeping the variable
 * marked as undefined by valgrind.
 */
#ifdef VALGRIND
#  define ERTS_UNDEF(V,I)
#else
# 201 "beam/sys.h"
#  define ERTS_UNDEF(V,I) V = I
#endif
# 203 "beam/sys.h"

/*
 * Compile time assert
 * (the actual compiler error msg can be a bit confusing)
 */
#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
# define ERTS_CT_ASSERT(e) \
    do { \
	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
    } while(0)
#else
# 214 "beam/sys.h"
# define ERTS_CT_ASSERT(e) \
    do { \
        enum { compile_time_assert__ = 1/(e) }; \
    } while (0)
#endif
# 219 "beam/sys.h"

/*
 * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
 * But MSC doesn't define __STDC__, unless we compile with the -Za
 * flag (strict ANSI C, no Microsoft extension).  Compiling with -Za
 * doesn't work: some Microsoft headers fail to compile...
 *
 * Solution: Test if __STDC__ or _MSC_VER is defined.
 *
 * Note: Simply defining __STDC__ doesn't work, as some Microsoft
 * headers will fail to compile!
 */

#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 232 "beam/sys.h"
# 233 "beam/sys.h"

/* This isn't sys-dependent, but putting it here benefits sys.c and drivers
   - allow use of 'const' regardless of compiler */

#if !defined(__STDC__) && !defined(_MSC_VER)
#  define const
#endif
# 240 "beam/sys.h"

#undef __deprecated
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
#  define __deprecated __attribute__((deprecated))
#else
# 245 "beam/sys.h"
#  define __deprecated
#endif
# 247 "beam/sys.h"
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
#  define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
#else
# 250 "beam/sys.h"
#  define erts_align_attribute(SZ)
#endif
# 252 "beam/sys.h"

/*
** Data types:
**
** Eterm: A tagged erlang term (possibly 64 bits)
** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
** UInt:  An unsigned integer exactly as large as an Eterm.
** SInt:  A signed integer exactly as large as an eterm and therefor large
**        enough to hold the return value of the signed_val() macro.
** UWord: An unsigned integer at least as large as a void * and also as large
**          or larger than an Eterm
** SWord: A signed integer at least as large as a void * and also as large
**          or larger than an Eterm
** Uint32: An unsigned integer of 32 bits exactly
** Sint32: A signed integer of 32 bits exactly
** Uint16: An unsigned integer of 16 bits exactly
** Sint16: A signed integer of 16 bits exactly.
*/

#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
#error Cannot handle this combination of int/long/void*/size_t sizes
#endif
# 276 "beam/sys.h"

#if SIZEOF_VOID_P == 8
#undef  ARCH_32
#define ARCH_64
#define ERTS_SIZEOF_TERM 8
#elif SIZEOF_VOID_P == 4
# 282 "beam/sys.h"
#define ARCH_32
#undef  ARCH_64
#define ERTS_SIZEOF_TERM 4
#else
# 286 "beam/sys.h"
#error Neither 32 nor 64 bit architecture
#endif
# 288 "beam/sys.h"
#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
#    define HALFWORD_HEAP 1
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND) ASSERT(COND)
#    undef ERTS_SIZEOF_TERM
#    define ERTS_SIZEOF_TERM 4
#else
# 295 "beam/sys.h"
#    define HALFWORD_HEAP 0
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND)
#endif
# 299 "beam/sys.h"

#if SIZEOF_VOID_P != SIZEOF_SIZE_T
#error sizeof(void*) != sizeof(size_t)
#endif
# 303 "beam/sys.h"

#if HALFWORD_HEAP

#if SIZEOF_INT == 4
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define ERTS_UINT_MAX UINT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#else
# 314 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 316 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long UWord;
typedef long          SWord;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#elif SIZEOF_VOID_P == SIZEOF_INT
# 325 "beam/sys.h"
typedef unsigned int UWord;
typedef int          SWord;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 332 "beam/sys.h"
typedef unsigned long long UWord;
typedef long long          SWord;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#else
# 339 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 341 "beam/sys.h"

#else /* !HALFWORD_HEAP */
# 343 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long Eterm;
typedef unsigned long Uint;
typedef long          Sint;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_INT
# 355 "beam/sys.h"
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 365 "beam/sys.h"
typedef unsigned long long Eterm;
typedef unsigned long long Uint;
typedef long long          Sint;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
#if defined(__WIN32__)
#define ErtsStrToSint _strtoi64
#else
# 376 "beam/sys.h"
#define ErtsStrToSint strtoll
#endif
# 378 "beam/sys.h"
#else
# 379 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 381 "beam/sys.h"

typedef Uint UWord;
typedef Sint SWord;
#define ERTS_UINT_MAX ERTS_UWORD_MAX

#endif /* HALFWORD_HEAP */
# 387 "beam/sys.h"

typedef UWord BeamInstr;

#ifndef HAVE_INT64
#  if SIZEOF_LONG == 8
#    define HAVE_INT64 1
typedef unsigned long Uint64;
typedef long          Sint64;
#    ifdef ULONG_MAX
#      define ERTS_UINT64_MAX ULONG_MAX
#    endif
# 398 "beam/sys.h"
#    ifdef LONG_MAX
#      define ERTS_SINT64_MAX LONG_MAX
#    endif
# 401 "beam/sys.h"
#    ifdef LONG_MIN
#      define ERTS_SINT64_MIN LONG_MIN
#    endif
# 404 "beam/sys.h"
#  elif SIZEOF_LONG_LONG == 8
# 405 "beam/sys.h"
#    define HAVE_INT64 1
typedef unsigned long long Uint64;
typedef long long          Sint64;
#    ifdef ULLONG_MAX
#      define ERTS_UINT64_MAX ULLONG_MAX
#    endif
# 411 "beam/sys.h"
#    ifdef LLONG_MAX
#      define ERTS_SINT64_MAX LLONG_MAX
#    endif
# 414 "beam/sys.h"
#    ifdef LLONG_MIN
#      define ERTS_SINT64_MIN LLONG_MIN
#    endif
# 417 "beam/sys.h"
#  else
# 418 "beam/sys.h"
#    error "No 64-bit integer type found"
#  endif
# 420 "beam/sys.h"
#endif
# 421 "beam/sys.h"

#ifndef ERTS_UINT64_MAX
#  define ERTS_UINT64_MAX (~((Uint64) 0))
#endif
# 425 "beam/sys.h"
#ifndef ERTS_SINT64_MAX
#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
#endif
# 428 "beam/sys.h"
#ifndef ERTS_SINT64_MIN
#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
#endif
# 431 "beam/sys.h"

#if SIZEOF_LONG == 4
typedef unsigned long Uint32;
typedef long          Sint32;
#elif SIZEOF_INT == 4
# 436 "beam/sys.h"
typedef unsigned int Uint32;
typedef int          Sint32;
#else
# 439 "beam/sys.h"
#error Found no appropriate type to use for 'Uint32' and 'Sint32'
#endif
# 441 "beam/sys.h"

#if SIZEOF_INT == 2
typedef unsigned int Uint16;
typedef int          Sint16;
#elif SIZEOF_SHORT == 2
# 446 "beam/sys.h"
typedef unsigned short Uint16;
typedef short          Sint16;
#else
# 449 "beam/sys.h"
#error Found no appropriate type to use for 'Uint16' and 'Sint16'
#endif
# 451 "beam/sys.h"

#if CHAR_BIT == 8
typedef unsigned char byte;
#else
# 455 "beam/sys.h"
#error Found no appropriate type to use for 'byte'
#endif
# 457 "beam/sys.h"

#if defined(ARCH_64) && !HAVE_INT64
#error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
#endif
# 461 "beam/sys.h"

#ifdef WORDS_BIGENDIAN
#  define ERTS_HUINT_HVAL_HIGH 0
#  define ERTS_HUINT_HVAL_LOW 1
#else
# 466 "beam/sys.h"
#  define ERTS_HUINT_HVAL_HIGH 1
#  define ERTS_HUINT_HVAL_LOW 0
#endif
# 469 "beam/sys.h"
#if ERTS_SIZEOF_TERM == 8
typedef union {
    Uint val;
    Uint32 hval[2];
} HUint;
#elif ERTS_SIZEOF_TERM == 4
# 475 "beam/sys.h"
typedef union {
    Uint val;
    Uint16 hval[2];
} HUint;
#else
# 480 "beam/sys.h"
#error "Unsupported size of term"
#endif
# 482 "beam/sys.h"

#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
    (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 486 "beam/sys.h"
# 487 "beam/sys.h"

/* needed by erl_smp.h */
int erts_send_warning_to_logger_str_nogl(char *);

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/sys.h"
# 492 "beam/sys.h"

#ifdef ERTS_WANT_BREAK_HANDLING
#  ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#    define ERTS_BREAK_REQUESTED \
  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
#  else
# 499 "beam/sys.h"
extern volatile int erts_break_requested;
#    define ERTS_BREAK_REQUESTED erts_break_requested
#  endif
# 502 "beam/sys.h"
void erts_do_break_handling(void);
#endif
# 504 "beam/sys.h"

#ifdef ERTS_WANT_GOT_SIGUSR1
#  ifndef UNIX
#    define ERTS_GOT_SIGUSR1 0
#  else
# 509 "beam/sys.h"
#    ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
#    else
# 513 "beam/sys.h"
extern volatile int erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
#    endif
# 516 "beam/sys.h"
#  endif
# 517 "beam/sys.h"
#endif
# 518 "beam/sys.h"

#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
extern erts_tsd_key_t erts_is_crash_dumping_key;
#define ERTS_SOMEONE_IS_CRASH_DUMPING \
  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
# 527 "beam/sys.h"
extern volatile int erts_writing_erl_crash_dump;
#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
# 531 "beam/sys.h"

/* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
   but be able to fall back on bcopy() etc on systems that don't have
   mem*(), but this doesn't work to well with memset()/bzero() - thus the
   memzero() macro.
*/

/* xxxP */
#if defined(USE_BCOPY)
#  define memcpy(a, b, c) bcopy((b), (a), (c))
#  define memcmp(a, b, c) bcmp((a), (b), (c))
#  define memzero(buf, len) bzero((buf), (len))
#else
# 544 "beam/sys.h"
#  define memzero(buf, len) memset((buf), '\0', (len))
#endif
# 546 "beam/sys.h"

/* Stuff that is useful for port programs, drivers, etc */

#ifdef ISC32			/* Too much for the Makefile... */
#  define signal	sigset
#  define NO_ASINH
#  define NO_ACOSH
#  define NO_ATANH
#  define NO_FTRUNCATE
#  define SIG_SIGHOLD
#  define _POSIX_SOURCE 
#  define _XOPEN_SOURCE
#endif
# 559 "beam/sys.h"

#ifdef QNX			/* Too much for the Makefile... */
#  define SYS_SELECT_H
#  define NO_ERF
#  define NO_ERFC
/* This definition doesn't take NaN into account, but matherr() gets those */
#  define isfinite(x) (fabs(x) != HUGE_VAL)
#  define USE_MATHERR
#  define HAVE_FINITE
#endif
# 569 "beam/sys.h"


#ifdef WANT_NONBLOCKING	    /* must define this to pull in fcntl.h/ioctl.h */

/* This is really a mess... We used to use fcntl O_NDELAY, but that seems
   to only work on SunOS 4 - in particular, on SysV-based systems
   (including Solaris 2), it does set non-blocking mode, but causes
   read() to return 0!!  fcntl O_NONBLOCK is specified by POSIX, and
   seems to work on most systems, with the notable exception of AIX,
   where the old ioctl FIONBIO is the *only* one that will set a *socket*
   in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
   pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */

#  ifdef __WIN32__

static unsigned long zero_value = 0, one_value = 1;
#    define SET_BLOCKING(fd)	{ if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); }
#    define SET_NONBLOCKING(fd)	ioctlsocket((fd), FIONBIO, &one_value)

#  else
# 589 "beam/sys.h"
#    ifdef NB_FIONBIO		/* Old BSD */
#if 0 /* expanded by -frewrite-includes */
#      include <sys/ioctl.h>
#endif /* expanded by -frewrite-includes */
# 590 "beam/sys.h"
# 591 "beam/sys.h"
  static const int zero_value = 0, one_value = 1;
#      define SET_BLOCKING(fd)         ioctl((fd), FIONBIO, &zero_value)
#      define SET_NONBLOCKING(fd)      ioctl((fd), FIONBIO, &one_value)
#      define ERRNO_BLOCK EWOULDBLOCK
#    else /* !NB_FIONBIO */
# 596 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#      include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 596 "beam/sys.h"
# 597 "beam/sys.h"
#      ifdef NB_O_NDELAY               /* Nothing needs this? */
#        define NB_FLAG O_NDELAY
#        ifndef ERRNO_BLOCK            /* allow override (e.g. EAGAIN) via Makefile */
#          define ERRNO_BLOCK EWOULDBLOCK
#        endif
# 602 "beam/sys.h"
#      else  /* !NB_O_NDELAY */	/* The True Way - POSIX!:-) */
# 603 "beam/sys.h"
#        define NB_FLAG O_NONBLOCK
#        define ERRNO_BLOCK EAGAIN
#      endif /* !NB_O_NDELAY */
# 606 "beam/sys.h"
#      define SET_BLOCKING(fd)         fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) & ~NB_FLAG)
#      define SET_NONBLOCKING(fd)      fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) | NB_FLAG)
#    endif /* !NB_FIONBIO */
# 611 "beam/sys.h"
#  endif /* !__WIN32__ */
# 612 "beam/sys.h"
#endif /* WANT_NONBLOCKING */
# 613 "beam/sys.h"

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);

/* Some special erts_exit() codes: */
#define ERTS_INTR_EXIT	-1		/* called from signal handler */
#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */

#define ERTS_INTERNAL_ERROR(What) \
    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
	     __FILE__, __LINE__, __func__, What)

Eterm erts_check_io_info(void *p);

/* Size of misc memory allocated from system dependent code */
Uint erts_sys_misc_mem_sz(void);

/* print stuff is declared here instead of in global.h, so sys stuff won't
   have to include global.h */
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 633 "beam/sys.h"
# 634 "beam/sys.h"

/* Io constants to erts_print and erts_putc */
#define ERTS_PRINT_STDERR	(2)
#define ERTS_PRINT_STDOUT	(1)
#define ERTS_PRINT_FILE		(-1)
#define ERTS_PRINT_SBUF		(-2)
#define ERTS_PRINT_SNBUF	(-3)
#define ERTS_PRINT_DSBUF	(-4)

#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF

typedef struct {
    char *buf;
    size_t size;
} erts_print_sn_buf;

int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
int erts_putc(int to, void *arg, char);			/* in utils.c */

/* logger stuff is declared here instead of in global.h, so sys files
   won't have to include global.h */

erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
int erts_send_info_to_logger_str(Eterm, char *); 
int erts_send_warning_to_logger_str(Eterm, char *);
int erts_send_error_to_logger_str(Eterm, char *);
int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
/* needed by erl_smp.h (declared above)
   int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);

typedef struct preload {
    char *name;			/* Name of module */
    int  size;			/* Size of code */
    unsigned char* code;	/* Code pointer */
} Preload;


/*
 * This structure contains options to all built in drivers.
 * None of the drivers use all of the fields.
 */

typedef struct _SysDriverOpts {
    Uint ifd;			/* Input file descriptor (fd driver). */
    Uint ofd;			/* Outputfile descriptor (fd driver). */
    int packet_bytes;		/* Number of bytes in packet header. */
    int read_write;		/* Read and write bits. */
    int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
    int redir_stderr;           /* Redirect stderr to stdout: TRUE/FALSE. */
    int hide_window;		/* Hide this windows (Windows). */
    int exit_status;		/* Report exit status of subprocess. */
    int overlapped_io;          /* Only has effect on windows NT et al */
    char *envir;		/* Environment of the port process, */
				/* in Windows format. */
    char **argv;                /* Argument vector in Unix'ish format. */
    char *wd;			/* Working directory. */
    unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
				   ERTS_SPAWN_EXTERNAL | both*/ 
    int parallelism;            /* Optimize for parallelism */
} SysDriverOpts;

extern char *erts_default_arg0;

extern char os_type[];

typedef struct {
    int have_os_monotonic_time;
    int have_corrected_os_monotonic_time;
    ErtsMonotonicTime os_monotonic_time_unit;
    ErtsMonotonicTime sys_clock_resolution;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
	int extended;
    } os_monotonic_time_info;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
    } os_system_time_info;
} ErtsSysInitTimeResult;

#define ERTS_SYS_INIT_TIME_RESULT_INITER \
    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}

extern void erts_init_sys_time_sup(void);
extern void sys_init_time(ErtsSysInitTimeResult *);
extern void erts_late_sys_init_time(void);
extern void erts_deliver_time(void);
extern void erts_time_remaining(SysTimeval *);
extern void erts_sys_init_float(void);
extern void erts_thread_init_float(void);
extern void erts_thread_disable_fpe(void);
ERTS_GLB_INLINE int erts_block_fpe(void);
ERTS_GLB_INLINE void erts_unblock_fpe(int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int erts_block_fpe(void)
{
    return erts_sys_block_fpe();
}

ERTS_GLB_INLINE void erts_unblock_fpe(int unmasked)
{
    erts_sys_unblock_fpe(unmasked);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 754 "beam/sys.h"


/* Dynamic library/driver loading */
typedef struct {
    char* str;
}ErtsSysDdllError;
#define ERTS_SYS_DDLL_ERROR_INIT {NULL}
extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*);
#define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL)
extern void *erts_sys_ddll_call_init(void *function);
extern void *erts_sys_ddll_call_nif_init(void *function);
extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*);
#define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL)
extern char *erts_sys_ddll_error(int code);



/*
 * System interfaces for startup.
 */
void erts_sys_schedule_interrupt(int set);
#ifdef ERTS_SMP
void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
void erts_sys_main_thread(void);
#endif
# 785 "beam/sys.h"

extern int erts_sys_prepare_crash_dump(int secs);
extern void erts_sys_pre_init(void);
extern void erl_sys_init(void);
extern void erl_sys_args(int *argc, char **argv);
extern void erl_sys_schedule(int);
void sys_tty_reset(int);

int sys_max_files(void);
void sys_init_io(void);
Preload* sys_preloaded(void);
unsigned char* sys_preload_begin(Preload*);
void sys_preload_end(Preload*);
int sys_get_key(int);
void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
		       UWord *ms_user_diff, UWord *ms_sys_diff);
void wall_clock_elapsed_time_both(UWord *ms_total, 
				  UWord *ms_diff);
void get_time(int *hour, int *minute, int *second);
void get_date(int *year, int *month, int *day);
void get_localtime(int *year, int *month, int *day, 
		   int *hour, int *minute, int *second);
void get_universaltime(int *year, int *month, int *day, 
		       int *hour, int *minute, int *second);
int seconds_to_univ(Sint64 seconds, 
		    Sint *year, Sint *month, Sint *day, 
		    Sint *hour, Sint *minute, Sint *second);
int univ_to_seconds(Sint year, Sint month, Sint day, 
		    Sint hour, Sint minute, Sint second,
		    Sint64* seconds);
int univ_to_local(
    Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second);
int local_to_univ(Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second, int isdst);
void get_now(Uint*, Uint*, Uint*);
struct ErtsSchedulerData_;
ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
ErtsMonotonicTime erts_get_time_offset(void);
void
erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
void get_sys_now(Uint*, Uint*, Uint*);
void set_break_quit(void (*)(void), void (*)(void));

void os_flavor(char*, unsigned);
void os_version(int*, int*, int*);
void init_getenv_state(GETENV_STATE *);
char * getenv_string(GETENV_STATE *);
void fini_getenv_state(GETENV_STATE *);

#define HAVE_ERTS_CHECK_IO_DEBUG
typedef struct {
    int no_used_fds;
    int no_driver_select_structs;
    int no_driver_event_structs;
} ErtsCheckIoDebugInfo;
int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);

int erts_sys_is_area_readable(char *start, char *stop);

/* xxxP */
#define SYS_DEFAULT_FLOAT_DECIMALS 20
void init_sys_float(void);
int sys_chars_to_double(char*, double*);
int sys_double_to_chars(double, char*, size_t);
int sys_double_to_chars_ext(double, char*, size_t, size_t);
int sys_double_to_chars_fast(double, char*, int, int, int);
void sys_get_pid(char *, size_t);

/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
int erts_sys_putenv(char *key, char *value);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_putenv_raw(char *key, char *value);
/* erts_sys_getenv() returns 0 on success (length of value string in
   *size), a value > 0 if value buffer is too small (*size is set to needed
   size), and a value < 0 on failure. */
int erts_sys_getenv(char *key, char *value, size_t *size);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_getenv_raw(char *key, char *value, size_t *size);
/* erts_sys_getenv__() is only allowed to be used in early init phase */
int erts_sys_getenv__(char *key, char *value, size_t *size);
/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
int erts_sys_unsetenv(char *key);

/* Easier to use, but not as efficient, environment functions */
char *erts_read_env(char *key);
void erts_free_read_env(void *value);

#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
extern void sys_thr_resume(erts_tid_t tid);
extern void sys_thr_suspend(erts_tid_t tid);
#endif
# 878 "beam/sys.h"

/* utils.c */

/* Options to sys_alloc_opt */
#define SYS_ALLOC_OPT_TRIM_THRESHOLD 0
#define SYS_ALLOC_OPT_TOP_PAD        1
#define SYS_ALLOC_OPT_MMAP_THRESHOLD 2
#define SYS_ALLOC_OPT_MMAP_MAX       3

/* Default values to sys_alloc_opt options */
#define ERTS_DEFAULT_TRIM_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_TOP_PAD         0
#define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_MMAP_MAX        64

int sys_alloc_opt(int, int);

typedef struct {
  int trim_threshold;
  int top_pad;
  int mmap_threshold;
  int mmap_max;
} SysAllocStat;

void sys_alloc_stat(SysAllocStat *);

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#undef ERTS_REFC_DEBUG
#define ERTS_REFC_DEBUG
#endif
# 908 "beam/sys.h"

typedef erts_smp_atomic_t erts_refc_t;

ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
				   erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
					   erts_aint_t min_val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}

ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 941 "beam/sys.h"
    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
# 943 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 955 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 968 "beam/sys.h"
    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
# 970 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 982 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
		 diff, val, min_val);
#else
# 995 "beam/sys.h"
    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
# 997 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 1009 "beam/sys.h"
    return val;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1013 "beam/sys.h"

#ifdef ERTS_ENABLE_KERNEL_POLL
extern int erts_use_kernel_poll;
#endif
# 1017 "beam/sys.h"

#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
#define sys_memset(s,c,n)    memset(s,c,n)
#define sys_memzero(s, n)    memset(s,'\0',n)
#define sys_strcmp(s1,s2)    strcmp(s1,s2)
#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
#define sys_strcpy(s1,s2)    strcpy(s1,s2)
#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
#define sys_strlen(s)        strlen(s)

/* define function symbols (needed in sys_drv_api) */
#define sys_fp_alloc     sys_alloc
#define sys_fp_realloc   sys_realloc
#define sys_fp_free      sys_free
#define sys_fp_memcpy    memcpy
#define sys_fp_memmove   memmove
#define sys_fp_memcmp    memcmp
#define sys_fp_memset    memset
/* #define sys_fp_memzero    elib_memzero */
#define sys_fp_strcmp    strcmp
#define sys_fp_strncmp   strncmp
#define sys_fp_strcpy    strcpy
#define sys_fp_strncpy   strncpy
#define sys_fp_strlen    strlen


/* Return codes from the nb_read and nb_write functions */
#define FD_READY 1
#define FD_CONTINUE 2
#define FD_ERROR 3



/* Standard set of integer macros  .. */

#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
                      (((Uint64)((unsigned char*) (s))[7])))

#define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                            ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
                            ((char*)(s))[2] = (char)((Sint64)(i) >> 40) & 0xff;\
                            ((char*)(s))[3] = (char)((Sint64)(i) >> 32) & 0xff;\
                            ((char*)(s))[4] = (char)((Sint64)(i) >> 24) & 0xff;\
                            ((char*)(s))[5] = (char)((Sint64)(i) >> 16) & 0xff;\
                            ((char*)(s))[6] = (char)((Sint64)(i) >> 8)  & 0xff;\
                            ((char*)(s))[7] = (char)((Sint64)(i))       & 0xff;\
                           } while (0) 

#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \
                      (((unsigned char*) (s))[1] << 16) | \
                      (((unsigned char*) (s))[2] << 8)  | \
                      (((unsigned char*) (s))[3]))

#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff;   \
                            ((char*)(s))[1] = (char)((i) >> 16) & 0xff;   \
                            ((char*)(s))[2] = (char)((i) >> 8)  & 0xff;   \
                            ((char*)(s))[3] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
                      (((unsigned char*) (s))[1] << 8)  | \
                      (((unsigned char*) (s))[2]))

#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                      (((unsigned char*)  (s))[1]))


#define put_int16(i, s) do {((char*)(s))[0] = (char)((i) >> 8) & 0xff;  \
                            ((char*)(s))[1] = (char)(i)        & 0xff;} \
                        while (0)

#define get_int8(s) ((((unsigned char*)  (s))[0] ))


#define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)


/*
 * Use DEBUGF as you would use printf, but use double parentheses:
 *
 *   DEBUGF(("Error: %s\n", error));
 *
 * The output will appear in a special console.
 */

#ifdef DEBUG
void erl_debug(char* format, ...);
void erl_bin_write(unsigned char *, int, int);

#  define DEBUGF(x) erl_debug x
#else
# 1121 "beam/sys.h"
#  define DEBUGF(x)
#endif
# 1123 "beam/sys.h"

#ifdef __WIN32__
#ifdef ARCH_64
#define ERTS_ALLOC_ALIGN_BYTES 16
#define ERTS_SMALL_ABS(Small) _abs64(Small) 
#else
# 1129 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1132 "beam/sys.h"
#else
# 1133 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1136 "beam/sys.h"

#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
#endif
# 1140 "beam/sys.h"

#ifdef __WIN32__
void call_break_handler(void);
char* last_error(void);
char* win32_errorstr(int);
#endif
# 1146 "beam/sys.h"

/************************************************************************
 * Find out the native filename encoding of the process (look at locale of 
 * Unix processes and just do UTF16 on windows 
 ************************************************************************/
#define ERL_FILENAME_UNKNOWN   (0)
#define ERL_FILENAME_LATIN1    (1)
#define ERL_FILENAME_UTF8      (2)
#define ERL_FILENAME_UTF8_MAC  (3)
#define ERL_FILENAME_WIN_WCHAR (4)

/************************************************************************
 * If a filename in for example list_dir is not in the right encoding, it
 * will be skipped in the resulting list, but depending on a startup setting
 * we will inform the user in different ways. These macros define the
 * different reactions to wrongly coded filenames. In the error case an
 * exception will be thrown by prim_file.
 ************************************************************************/
#define ERL_FILENAME_WARNING_WARNING (0)
#define ERL_FILENAME_WARNING_IGNORE (1)
#define ERL_FILENAME_WARNING_ERROR (2)

/***********************************************************************
 * The user can request a range of character that he/she consider
 * printable. Currently this can be either latin1 or unicode, but
 * in the future a set of ranges, or languages, could be specified.
 ***********************************************************************/
#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0)
#define ERL_PRINTABLE_CHARACTERS_UNICODE (1)

int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
void erts_set_user_requested_filename_encoding(int encoding, int warning);
int erts_get_user_requested_filename_encoding(void);
int erts_get_filename_warning_type(void);
/* This function is called from erl_init. The setting is read by BIF's 
   in io/io_lib. Setting is not atomic. */
void erts_set_printable_characters(int range);
/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */
int erts_get_printable_characters(void);

void erts_init_sys_common_misc(void);

#endif
# 1190 "beam/sys.h"
# 261 "beam/erl_threads.h" 2

#ifdef USE_THREADS

#define ETHR_TRY_INLINE_FUNCS
#if 0 /* expanded by -frewrite-includes */
#include "ethread.h"
#endif /* expanded by -frewrite-includes */
# 265 "beam/erl_threads.h"
# 1 "../include/internal/ethread.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2004-2015. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread library for use in the ERTS and other OTP
 *              applications.
 * Author: Rickard Green
 */

#ifndef ETHREAD_H__
#define ETHREAD_H__

#ifndef ETHR_HAVE_ETHREAD_DEFINES
#if 0 /* expanded by -frewrite-includes */
#  include "ethread_header_config.h"
#endif /* expanded by -frewrite-includes */
# 31 "../include/internal/ethread.h"
# 32 "../include/internal/ethread.h"
#endif
# 33 "../include/internal/ethread.h"

#if 0 /* expanded by -frewrite-includes */
#include <stdlib.h>
#endif /* expanded by -frewrite-includes */
# 34 "../include/internal/ethread.h"
# 35 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include "ethread_inline.h"
#endif /* expanded by -frewrite-includes */
# 35 "../include/internal/ethread.h"
# 1 "../include/internal/ethread_inline.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2004-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ETHREAD_INLINE_H__
#define ETHREAD_INLINE_H__

#define ETHR_GCC_COMPILER_FALSE 0 /* Not a gcc compatible compiler */
#define ETHR_GCC_COMPILER_TRUE 1 /* The GNU gcc compiler */
/* Negative integers for gcc compatible compilers */
#define ETHR_GCC_COMPILER_CLANG -1 /* The Clang gcc compatible compiler */
#define ETHR_GCC_COMPILER_ICC -2 /* The Intel gcc compatible compiler */
/* Line them up... */

/*
 * Unfortunately there is no easy and certain way of
 * detecting a true gcc compiler, since the compatible
 * ones all define the same defines as the true gnu-gcc...
 */
#if !defined(__GNUC__) && !defined(__GNUG__) 
#  define ETHR_GCC_COMPILER ETHR_GCC_COMPILER_FALSE
#elif defined(__clang__)
# 39 "../include/internal/ethread_inline.h"
#  define ETHR_GCC_COMPILER ETHR_GCC_COMPILER_CLANG
#elif defined(__ICC) || defined(__INTEL_COMPILER)
# 41 "../include/internal/ethread_inline.h"
#  define ETHR_GCC_COMPILER ETHR_GCC_COMPILER_ICC
#else
# 43 "../include/internal/ethread_inline.h"
/* Seems to be the true gnu-gcc... */
#  define ETHR_GCC_COMPILER ETHR_GCC_COMPILER_TRUE
#endif
# 46 "../include/internal/ethread_inline.h"

#if !defined(__GNUC__)
#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 50 "../include/internal/ethread_inline.h"
#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 53 "../include/internal/ethread_inline.h"
#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 56 "../include/internal/ethread_inline.h"
#  define ETHR_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 59 "../include/internal/ethread_inline.h"

#undef ETHR_INLINE
#if defined(__GNUC__)
#  define ETHR_INLINE __inline__
#  if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1)
#    define ETHR_FORCE_INLINE __inline__ __attribute__((__always_inline__))
#  else
# 66 "../include/internal/ethread_inline.h"
#    define ETHR_FORCE_INLINE __inline__
#  endif
# 68 "../include/internal/ethread_inline.h"
#elif defined(__WIN32__)
# 69 "../include/internal/ethread_inline.h"
#  define ETHR_INLINE __forceinline
#  define ETHR_FORCE_INLINE __forceinline
#endif
# 72 "../include/internal/ethread_inline.h"

#endif /* #ifndef ETHREAD_INLINE_H__ */
# 74 "../include/internal/ethread_inline.h"
# 36 "../include/internal/ethread.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_errno.h"
#endif /* expanded by -frewrite-includes */
# 36 "../include/internal/ethread.h"
# 37 "../include/internal/ethread.h"

#if defined(DEBUG)
#  define ETHR_DEBUG
#endif
# 41 "../include/internal/ethread.h"

#if defined(__PPC__) || defined (__POWERPC)
/* OSE compiler should be fixed! */
#define __ppc__
#endif
# 46 "../include/internal/ethread.h"

#if defined(ETHR_DEBUG)
#  undef ETHR_XCHK
#  define  ETHR_XCHK 1
#else
# 51 "../include/internal/ethread.h"
#  ifndef ETHR_XCHK
#    define ETHR_XCHK 0
#  endif
# 54 "../include/internal/ethread.h"
#endif
# 55 "../include/internal/ethread.h"

#if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \
    || (defined(__GNUC__) && defined(ERTS_MIXED_CYGWIN_VC)) \
    || (defined(__GNUC__) && defined(ERTS_MIXED_MSYS_VC))
#  undef ETHR_INLINE
#  define ETHR_INLINE 
#  undef ETHR_FORCE_INLINE
#  define ETHR_FORCE_INLINE
#  undef ETHR_TRY_INLINE_FUNCS
#endif
# 65 "../include/internal/ethread.h"

/* Assume 64-byte cache line size */
#define ETHR_CACHE_LINE_SIZE ASSUMED_CACHE_LINE_SIZE
#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1)

#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \
  (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE)

#ifndef ETHR_INLINE_FUNC_NAME_
#  define ETHR_INLINE_FUNC_NAME_(X) X
#endif
# 76 "../include/internal/ethread.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 82 "../include/internal/ethread.h"
#      define __func__ __FUNCTION__
#    endif
# 84 "../include/internal/ethread.h"
#  endif
# 85 "../include/internal/ethread.h"
#endif
# 86 "../include/internal/ethread.h"

int ethr_assert_failed(const char *file, int line, const char *func, char *a);
#ifdef ETHR_DEBUG
#define ETHR_ASSERT(A) \
  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A)))
#else
# 92 "../include/internal/ethread.h"
#define ETHR_ASSERT(A) ((void) 1)
#endif
# 94 "../include/internal/ethread.h"

#if defined(__GNUC__)
#  define ETHR_PROTO_NORETURN__ void __attribute__((noreturn))
#  define ETHR_IMPL_NORETURN__ void
#elif defined(__WIN32__) && defined(_MSC_VER)
# 99 "../include/internal/ethread.h"
#  define ETHR_PROTO_NORETURN__ __declspec(noreturn) void
#  define ETHR_IMPL_NORETURN__ __declspec(noreturn) void
#else
# 102 "../include/internal/ethread.h"
#  define ETHR_PROTO_NORETURN__ void
#  define ETHR_IMPL_NORETURN__ void
#endif
# 105 "../include/internal/ethread.h"

#if defined(ETHR_PTHREADS)
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * The pthread implementation                                                *
\*                                                                           */

#if defined(__linux__) && !defined(_GNU_SOURCE)
#error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE."
#endif
# 114 "../include/internal/ethread.h"

#if defined(ETHR_NEED_NPTL_PTHREAD_H)
#if 0 /* expanded by -frewrite-includes */
#include <nptl/pthread.h>
#endif /* expanded by -frewrite-includes */
# 116 "../include/internal/ethread.h"
# 117 "../include/internal/ethread.h"
#elif defined(ETHR_HAVE_MIT_PTHREAD_H)
# 118 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include <pthread/mit/pthread.h>
#endif /* expanded by -frewrite-includes */
# 118 "../include/internal/ethread.h"
# 119 "../include/internal/ethread.h"
#elif defined(ETHR_HAVE_PTHREAD_H)
# 120 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include <pthread.h>
#endif /* expanded by -frewrite-includes */
# 120 "../include/internal/ethread.h"
# 1 "/usr/include/pthread.h" 1 3 4
/*
 * Copyright (c) 1993, 1994 by Chris Provenzano, proven@mit.edu
 * Copyright (c) 1995-1998 by John Birrell <jb@cimlogic.com.au>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *  This product includes software developed by Chris Provenzano.
 * 4. The name of Chris Provenzano may not be used to endorse or promote 
 *	  products derived from this software without specific prior written
 *	  permission.
 *
 * THIS SOFTWARE IS PROVIDED BY CHRIS PROVENZANO ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL CHRIS PROVENZANO BE LIABLE FOR ANY 
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 
 * SUCH DAMAGE.
 *
 * $FreeBSD: head/include/pthread.h 296162 2016-02-28 17:52:33Z kib $
 */
#ifndef _PTHREAD_H_
#define	_PTHREAD_H_

/*
 * Header files.
 */
#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 41 "/usr/include/pthread.h" 3 4
# 42 "/usr/include/pthread.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_pthreadtypes.h>
#endif /* expanded by -frewrite-includes */
# 42 "/usr/include/pthread.h" 3 4
# 43 "/usr/include/pthread.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/_limits.h>
#endif /* expanded by -frewrite-includes */
# 43 "/usr/include/pthread.h" 3 4
# 44 "/usr/include/pthread.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <machine/_types.h>
#endif /* expanded by -frewrite-includes */
# 44 "/usr/include/pthread.h" 3 4
# 45 "/usr/include/pthread.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_sigset.h>
#endif /* expanded by -frewrite-includes */
# 45 "/usr/include/pthread.h" 3 4
# 46 "/usr/include/pthread.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sched.h>
#endif /* expanded by -frewrite-includes */
# 46 "/usr/include/pthread.h" 3 4
# 1 "/usr/include/sched.h" 1 3 4
/*-
 * Copyright (c) 1996, 1997
 *      HD Associates, Inc.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *      This product includes software developed by HD Associates, Inc
 *      and Jukka Antero Ukkonen.
 * 4. Neither the name of the author nor the names of any co-contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY HD ASSOCIATES AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL HD ASSOCIATES OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

/*-
 * Copyright (c) 2002-2008, Jeffrey Roberson <jeff@freebsd.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: head/sys/sys/sched.h 298145 2016-04-17 11:04:27Z kib $
 */

#ifndef _SCHED_H_
#define	_SCHED_H_

#ifdef _KERNEL
/*
 * General scheduling info.
 *
 * sched_load:
 *	Total runnable non-ithread threads in the system.
 *
 * sched_runnable:
 *	Runnable threads for this processor.
 */
int	sched_load(void);
int	sched_rr_interval(void);
int	sched_runnable(void);

/* 
 * Proc related scheduling hooks.
 */
void	sched_exit(struct proc *p, struct thread *childtd);
void	sched_fork(struct thread *td, struct thread *childtd);
void	sched_fork_exit(struct thread *td);
void	sched_class(struct thread *td, int class);
void	sched_nice(struct proc *p, int nice);

/*
 * Threads are switched in and out, block on resources, have temporary
 * priorities inherited from their procs, and use up cpu time.
 */
void	sched_exit_thread(struct thread *td, struct thread *child);
u_int	sched_estcpu(struct thread *td);
void	sched_fork_thread(struct thread *td, struct thread *child);
void	sched_lend_prio(struct thread *td, u_char prio);
void	sched_lend_user_prio(struct thread *td, u_char pri);
fixpt_t	sched_pctcpu(struct thread *td);
void	sched_prio(struct thread *td, u_char prio);
void	sched_sleep(struct thread *td, int prio);
void	sched_switch(struct thread *td, struct thread *newtd, int flags);
void	sched_throw(struct thread *td);
void	sched_unlend_prio(struct thread *td, u_char prio);
void	sched_user_prio(struct thread *td, u_char prio);
void	sched_userret(struct thread *td);
void	sched_wakeup(struct thread *td);
#ifdef	RACCT
#ifdef	SCHED_4BSD
fixpt_t	sched_pctcpu_delta(struct thread *td);
#endif
# 110 "/usr/include/sched.h" 3 4
#endif
# 111 "/usr/include/sched.h" 3 4

/*
 * Threads are moved on and off of run queues
 */
void	sched_add(struct thread *td, int flags);
void	sched_clock(struct thread *td);
void	sched_preempt(struct thread *td);
void	sched_rem(struct thread *td);
void	sched_relinquish(struct thread *td);
struct thread *sched_choose(void);
void	sched_idletd(void *);

/*
 * Binding makes cpu affinity permanent while pinning is used to temporarily
 * hold a thread on a particular CPU.
 */
void	sched_bind(struct thread *td, int cpu);
static __inline void sched_pin(void);
void	sched_unbind(struct thread *td);
static __inline void sched_unpin(void);
int	sched_is_bound(struct thread *td);
void	sched_affinity(struct thread *td);

/*
 * These procedures tell the process data structure allocation code how
 * many bytes to actually allocate.
 */
int	sched_sizeof_proc(void);
int	sched_sizeof_thread(void);

/*
 * This routine provides a consistent thread name for use with KTR graphing
 * functions.
 */
char	*sched_tdname(struct thread *td);
#ifdef KTR
void	sched_clear_tdname(struct thread *td);
#endif
# 149 "/usr/include/sched.h" 3 4

static __inline void
sched_pin(void)
{
	curthread->td_pinned++;
	__compiler_membar();
}

static __inline void
sched_unpin(void)
{
	__compiler_membar();
	curthread->td_pinned--;
}

/* sched_add arguments (formerly setrunqueue) */
#define	SRQ_BORING	0x0000		/* No special circumstances. */
#define	SRQ_YIELDING	0x0001		/* We are yielding (from mi_switch). */
#define	SRQ_OURSELF	0x0002		/* It is ourself (from mi_switch). */
#define	SRQ_INTR	0x0004		/* It is probably urgent. */
#define	SRQ_PREEMPTED	0x0008		/* has been preempted.. be kind */
#define	SRQ_BORROWING	0x0010		/* Priority updated due to prio_lend */

/* Scheduler stats. */
#ifdef SCHED_STATS
DPCPU_DECLARE(long, sched_switch_stats[SWT_COUNT]);

#define	SCHED_STAT_DEFINE_VAR(name, ptr, descr)				\
static void name ## _add_proc(void *dummy __unused)			\
{									\
									\
	SYSCTL_ADD_PROC(NULL,						\
	    SYSCTL_STATIC_CHILDREN(_kern_sched_stats), OID_AUTO,	\
	    #name, CTLTYPE_LONG|CTLFLAG_RD|CTLFLAG_MPSAFE,		\
	    ptr, 0, sysctl_dpcpu_long, "LU", descr);			\
}									\
SYSINIT(name, SI_SUB_LAST, SI_ORDER_MIDDLE, name ## _add_proc, NULL);

#define	SCHED_STAT_DEFINE(name, descr)					\
    DPCPU_DEFINE(unsigned long, name);					\
    SCHED_STAT_DEFINE_VAR(name, &DPCPU_NAME(name), descr)
/*
 * Sched stats are always incremented in critical sections so no atomic
 * is necesssary to increment them.
 */
#define SCHED_STAT_INC(var)     DPCPU_GET(var)++;
#else
# 196 "/usr/include/sched.h" 3 4
#define	SCHED_STAT_DEFINE_VAR(name, descr, ptr)
#define	SCHED_STAT_DEFINE(name, descr)
#define SCHED_STAT_INC(var)			(void)0
#endif
# 200 "/usr/include/sched.h" 3 4

/*
 * Fixup scheduler state for proc0 and thread0
 */
void schedinit(void);
#endif /* _KERNEL */
# 206 "/usr/include/sched.h" 3 4

/* POSIX 1003.1b Process Scheduling */

/*
 * POSIX scheduling policies
 */
#define SCHED_FIFO      1
#define SCHED_OTHER     2
#define SCHED_RR        3

struct sched_param {
        int     sched_priority;
};

/*
 * POSIX scheduling declarations for userland.
 */
#ifndef _KERNEL
#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 224 "/usr/include/sched.h" 3 4
# 225 "/usr/include/sched.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 225 "/usr/include/sched.h" 3 4
# 226 "/usr/include/sched.h" 3 4

#ifndef _PID_T_DECLARED
typedef __pid_t         pid_t;
#define _PID_T_DECLARED
#endif
# 231 "/usr/include/sched.h" 3 4

struct timespec;

__BEGIN_DECLS
int     sched_get_priority_max(int);
int     sched_get_priority_min(int);
int     sched_getparam(pid_t, struct sched_param *);
int     sched_getscheduler(pid_t);
int     sched_rr_get_interval(pid_t, struct timespec *);
int     sched_setparam(pid_t, const struct sched_param *);
int     sched_setscheduler(pid_t, int, const struct sched_param *);
int     sched_yield(void);
__END_DECLS

#endif
# 246 "/usr/include/sched.h" 3 4
#endif /* !_SCHED_H_ */
# 247 "/usr/include/sched.h" 3 4
# 47 "/usr/include/pthread.h" 2 3 4
#if 0 /* expanded by -frewrite-includes */
#include <time.h>
#endif /* expanded by -frewrite-includes */
# 47 "/usr/include/pthread.h" 3 4
# 48 "/usr/include/pthread.h" 3 4

/*
 * Run-time invariant values:
 */
#define	PTHREAD_DESTRUCTOR_ITERATIONS		4
#define	PTHREAD_KEYS_MAX			256
#define	PTHREAD_STACK_MIN			__MINSIGSTKSZ
#define	PTHREAD_THREADS_MAX			__ULONG_MAX
#define	PTHREAD_BARRIER_SERIAL_THREAD		-1

/*
 * Flags for threads and thread attributes.
 */
#define	PTHREAD_DETACHED		0x1
#define	PTHREAD_SCOPE_SYSTEM		0x2
#define	PTHREAD_INHERIT_SCHED		0x4
#define	PTHREAD_NOFLOAT			0x8

#define	PTHREAD_CREATE_DETACHED		PTHREAD_DETACHED
#define	PTHREAD_CREATE_JOINABLE		0
#define	PTHREAD_SCOPE_PROCESS		0
#define	PTHREAD_EXPLICIT_SCHED		0

/*
 * Values for process shared/private attributes.
 */
#define	PTHREAD_PROCESS_PRIVATE		0
#define	PTHREAD_PROCESS_SHARED		1

/*
 * Flags for cancelling threads
 */
#define	PTHREAD_CANCEL_ENABLE		0
#define	PTHREAD_CANCEL_DISABLE		1
#define	PTHREAD_CANCEL_DEFERRED		0
#define	PTHREAD_CANCEL_ASYNCHRONOUS	2
#define	PTHREAD_CANCELED		((void *) 1)

/*
 * Flags for once initialization.
 */
#define	PTHREAD_NEEDS_INIT	0
#define	PTHREAD_DONE_INIT	1

/*
 * Static once initialization values.
 */
#define	PTHREAD_ONCE_INIT	{ PTHREAD_NEEDS_INIT, NULL }

/*
 * Static initialization values.
 */
#define	PTHREAD_MUTEX_INITIALIZER	NULL
#define	PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP	((pthread_mutex_t)1)
#define	PTHREAD_COND_INITIALIZER	NULL
#define	PTHREAD_RWLOCK_INITIALIZER	NULL

/*
 * Default attribute arguments (draft 4, deprecated).
 */
#ifndef PTHREAD_KERNEL
#define	pthread_condattr_default	NULL
#define	pthread_mutexattr_default	NULL
#define	pthread_attr_default		NULL
#endif
# 113 "/usr/include/pthread.h" 3 4

#define	PTHREAD_PRIO_NONE	0
#define	PTHREAD_PRIO_INHERIT	1
#define	PTHREAD_PRIO_PROTECT	2

/*
 * Mutex types (Single UNIX Specification, Version 2, 1997).
 *
 * Note that a mutex attribute with one of the following types:
 *
 *	PTHREAD_MUTEX_NORMAL
 *	PTHREAD_MUTEX_RECURSIVE
 *
 * will deviate from POSIX specified semantics.
 */
enum pthread_mutextype {
	PTHREAD_MUTEX_ERRORCHECK	= 1,	/* Default POSIX mutex */
	PTHREAD_MUTEX_RECURSIVE		= 2,	/* Recursive mutex */
	PTHREAD_MUTEX_NORMAL		= 3,	/* No error checking */
	PTHREAD_MUTEX_ADAPTIVE_NP	= 4,	/* Adaptive mutex, spins briefly before blocking on lock */
	PTHREAD_MUTEX_TYPE_MAX
};

#define	PTHREAD_MUTEX_DEFAULT		PTHREAD_MUTEX_ERRORCHECK

struct _pthread_cleanup_info {
	__uintptr_t	pthread_cleanup_pad[8];
};

/*
 * Thread function prototype definitions:
 */
__BEGIN_DECLS
int		pthread_atfork(void (*)(void), void (*)(void), void (*)(void));
int		pthread_attr_destroy(pthread_attr_t *) __nonnull(1);
int		pthread_attr_getstack(const pthread_attr_t * __restrict, 
			void ** __restrict, size_t * __restrict)
			__nonnull_all;
int		pthread_attr_getstacksize(const pthread_attr_t *, size_t *)
			__nonnull_all;
int		pthread_attr_getguardsize(const pthread_attr_t *, size_t *);
int		pthread_attr_getstackaddr(const pthread_attr_t *, void **);
int		pthread_attr_getdetachstate(const pthread_attr_t *, int *)
			__nonnull_all;
int		pthread_attr_init(pthread_attr_t *) __nonnull(1);
int		pthread_attr_setstacksize(pthread_attr_t *, size_t)
			__nonnull(1);
int		pthread_attr_setguardsize(pthread_attr_t *, size_t)
			__nonnull(1);
int		pthread_attr_setstack(pthread_attr_t *, void *, size_t)
			__nonnull(1);
int		pthread_attr_setstackaddr(pthread_attr_t *, void *);
int		pthread_attr_setdetachstate(pthread_attr_t *, int) __nonnull(1);
int		pthread_barrier_destroy(pthread_barrier_t *);
int		pthread_barrier_init(pthread_barrier_t *,
			const pthread_barrierattr_t *, unsigned);
int		pthread_barrier_wait(pthread_barrier_t *);
int		pthread_barrierattr_destroy(pthread_barrierattr_t *);
int		pthread_barrierattr_getpshared(const pthread_barrierattr_t *,
			int *);
int		pthread_barrierattr_init(pthread_barrierattr_t *) __nonnull(1);
int		pthread_barrierattr_setpshared(pthread_barrierattr_t *, int);

#define		pthread_cleanup_push(cleanup_routine, cleanup_arg)		\
		{								\
			struct _pthread_cleanup_info __cleanup_info__;		\
			__pthread_cleanup_push_imp(cleanup_routine, cleanup_arg,\
				&__cleanup_info__);				\
			{

#define		pthread_cleanup_pop(execute)					\
				(void)0;					\
			}							\
			__pthread_cleanup_pop_imp(execute);			\
		}

int		pthread_condattr_destroy(pthread_condattr_t *) __nonnull(1);
int		pthread_condattr_getclock(const pthread_condattr_t *,
			clockid_t *) __nonnull_all;
int		pthread_condattr_getpshared(const pthread_condattr_t *, int *)
			__nonnull_all;
int		pthread_condattr_init(pthread_condattr_t *) __nonnull(1);
int		pthread_condattr_setclock(pthread_condattr_t *, clockid_t)
			__nonnull(1);
int		pthread_condattr_setpshared(pthread_condattr_t *, int)
			__nonnull(1);
int		pthread_cond_broadcast(pthread_cond_t *)
			__nonnull(1);
int		pthread_cond_destroy(pthread_cond_t *)
			__nonnull(1);
int		pthread_cond_init(pthread_cond_t *,
			const pthread_condattr_t *) __nonnull(1);
int		pthread_cond_signal(pthread_cond_t *) __nonnull(1);
int		pthread_cond_timedwait(pthread_cond_t *,
			pthread_mutex_t *__mutex, const struct timespec *)
			__nonnull_all __requires_exclusive(*__mutex);
int		pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *__mutex)
			__nonnull_all __requires_exclusive(*__mutex);
int		pthread_create(pthread_t *, const pthread_attr_t *,
			void *(*) (void *), void *) __nonnull(1) __nonnull(3);
int		pthread_detach(pthread_t);
int		pthread_equal(pthread_t, pthread_t);
void		pthread_exit(void *) __dead2;
void		*pthread_getspecific(pthread_key_t);
int		pthread_getcpuclockid(pthread_t, clockid_t *) __nonnull(2);
int		pthread_join(pthread_t, void **);
int		pthread_key_create(pthread_key_t *,
			void (*) (void *)) __nonnull(1);
int		pthread_key_delete(pthread_key_t);
int		pthread_mutexattr_init(pthread_mutexattr_t *) __nonnull(1);
int		pthread_mutexattr_destroy(pthread_mutexattr_t *) __nonnull(1);
int		pthread_mutexattr_getpshared(const pthread_mutexattr_t *,
			int *) __nonnull_all;
int		pthread_mutexattr_gettype(pthread_mutexattr_t *, int *)
			__nonnull_all;
int		pthread_mutexattr_settype(pthread_mutexattr_t *, int)
			__nonnull(1);
int		pthread_mutexattr_setpshared(pthread_mutexattr_t *, int)
			__nonnull(1);
int		pthread_mutex_destroy(pthread_mutex_t *__mutex)
			__nonnull(1) __requires_unlocked(*__mutex);
int		pthread_mutex_init(pthread_mutex_t *__mutex,
			const pthread_mutexattr_t *)
			__nonnull(1) __requires_unlocked(*__mutex);
int		pthread_mutex_lock(pthread_mutex_t *__mutex)
			__nonnull(1) __locks_exclusive(*__mutex);
int		pthread_mutex_trylock(pthread_mutex_t *__mutex)
			__nonnull(1) __trylocks_exclusive(0, *__mutex);
int		pthread_mutex_timedlock(pthread_mutex_t *__mutex,
			const struct timespec *)
			__nonnull_all __trylocks_exclusive(0, *__mutex);
int		pthread_mutex_unlock(pthread_mutex_t *__mutex)
			__nonnull(1) __unlocks(*__mutex);
int		pthread_once(pthread_once_t *, void (*) (void)) __nonnull_all;
int		pthread_rwlock_destroy(pthread_rwlock_t *__rwlock)
			__nonnull(1) __requires_unlocked(*__rwlock);
int		pthread_rwlock_init(pthread_rwlock_t *__rwlock,
			const pthread_rwlockattr_t *)
			__nonnull(1) __requires_unlocked(*__rwlock);
int		pthread_rwlock_rdlock(pthread_rwlock_t *__rwlock)
			__nonnull(1) __locks_shared(*__rwlock);
int		pthread_rwlock_timedrdlock(pthread_rwlock_t *__rwlock,
			const struct timespec *)
			__nonnull_all __trylocks_shared(0, *__rwlock);
int		pthread_rwlock_timedwrlock(pthread_rwlock_t *__rwlock,
			const struct timespec *)
			__nonnull_all __trylocks_exclusive(0, *__rwlock);
int		pthread_rwlock_tryrdlock(pthread_rwlock_t *__rwlock)
			__nonnull(1) __trylocks_shared(0, *__rwlock);
int		pthread_rwlock_trywrlock(pthread_rwlock_t *__rwlock)
			__nonnull(1) __trylocks_exclusive(0, *__rwlock);
int		pthread_rwlock_unlock(pthread_rwlock_t *__rwlock)
			__nonnull(1) __unlocks(*__rwlock);
int		pthread_rwlock_wrlock(pthread_rwlock_t *__rwlock)
			__nonnull(1) __locks_exclusive(*__rwlock);
int		pthread_rwlockattr_destroy(pthread_rwlockattr_t *)
			__nonnull(1);
int		pthread_rwlockattr_getkind_np(const pthread_rwlockattr_t *,
			int *);
int		pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *,
			int *) __nonnull_all;
int		pthread_rwlockattr_init(pthread_rwlockattr_t *)
			__nonnull(1);
int		pthread_rwlockattr_setkind_np(pthread_rwlockattr_t *, int);
int		pthread_rwlockattr_setpshared(pthread_rwlockattr_t *, int)
			__nonnull(1);
pthread_t	pthread_self(void);
int		pthread_setspecific(pthread_key_t, const void *);

int		pthread_spin_init(pthread_spinlock_t *__spin, int)
			__requires_unlocked(*__spin);
int		pthread_spin_destroy(pthread_spinlock_t *__spin)
			__requires_unlocked(*__spin);
int		pthread_spin_lock(pthread_spinlock_t *__spin)
			__locks_exclusive(*__spin);
int		pthread_spin_trylock(pthread_spinlock_t *__spin)
			__trylocks_exclusive(0, *__spin);
int		pthread_spin_unlock(pthread_spinlock_t *__spin)
			__unlocks(*__spin);
int		pthread_cancel(pthread_t);
int		pthread_setcancelstate(int, int *);
int		pthread_setcanceltype(int, int *);
void		pthread_testcancel(void);

#if __BSD_VISIBLE
int		pthread_getprio(pthread_t);
int		pthread_setprio(pthread_t, int);
void		pthread_yield(void);
#endif
# 302 "/usr/include/pthread.h" 3 4

int		pthread_mutexattr_getprioceiling(pthread_mutexattr_t *,
			int *);
int		pthread_mutexattr_setprioceiling(pthread_mutexattr_t *,
			int);
int		pthread_mutex_getprioceiling(pthread_mutex_t *, int *);
int		pthread_mutex_setprioceiling(pthread_mutex_t *, int, int *);

int		pthread_mutexattr_getprotocol(pthread_mutexattr_t *, int *);
int		pthread_mutexattr_setprotocol(pthread_mutexattr_t *, int);

int		pthread_attr_getinheritsched(const pthread_attr_t *, int *);
int		pthread_attr_getschedparam(const pthread_attr_t *,
			struct sched_param *) __nonnull_all;
int		pthread_attr_getschedpolicy(const pthread_attr_t *, int *)
			__nonnull_all;
int		pthread_attr_getscope(const pthread_attr_t *, int *)
			__nonnull_all;
int		pthread_attr_setinheritsched(pthread_attr_t *, int);
int		pthread_attr_setschedparam(pthread_attr_t *,
			const struct sched_param *) __nonnull(1) __nonnull(2);
int		pthread_attr_setschedpolicy(pthread_attr_t *, int) __nonnull(1);
int		pthread_attr_setscope(pthread_attr_t *, int) __nonnull(1);
int		pthread_getschedparam(pthread_t pthread, int *,
			struct sched_param *) __nonnull(2) __nonnull(3);
int		pthread_setschedparam(pthread_t, int,
			const struct sched_param *) __nonnull(3);
#if __XSI_VISIBLE
int		pthread_getconcurrency(void);
int		pthread_setconcurrency(int);
#endif
# 333 "/usr/include/pthread.h" 3 4

void		__pthread_cleanup_push_imp(void (*)(void *), void *,
			struct _pthread_cleanup_info *);
void		__pthread_cleanup_pop_imp(int);
__END_DECLS

#endif
# 340 "/usr/include/pthread.h" 3 4
# 121 "../include/internal/ethread.h" 2
#endif
# 122 "../include/internal/ethread.h"

/* Types */

typedef pthread_t ethr_tid;

typedef pthread_key_t ethr_tsd_key;

#define ETHR_HAVE_ETHR_SIG_FUNCS 1

#if defined(PURIFY) || defined(VALGRIND)
#  define ETHR_FORCE_PTHREAD_RWLOCK
#  define ETHR_FORCE_PTHREAD_MUTEX
#endif
# 135 "../include/internal/ethread.h"

#if !defined(ETHR_FORCE_PTHREAD_RWLOCK)
#  define ETHR_USE_OWN_RWMTX_IMPL__
#endif
# 139 "../include/internal/ethread.h"

#if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0
#  define ETHR_USE_OWN_MTX_IMPL__
#endif
# 143 "../include/internal/ethread.h"

#elif defined(ETHR_WIN32_THREADS)
# 145 "../include/internal/ethread.h"
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * The native win32 threads implementation                                   *
\*                                                                           */

#if !defined(_WIN32_WINNT)
#error "_WIN32_WINNT not defined. Please, compile all files with -D_WIN32_WINNT=0x0403"
#elif _WIN32_WINNT < 0x0403
# 152 "../include/internal/ethread.h"
#error "_WIN32_WINNT defined to a value less than 0x0403. Please, compile all files with -D_WIN32_WINNT=0x0403"
#endif
# 154 "../include/internal/ethread.h"

#ifdef WIN32_LEAN_AND_MEAN
#  define ETHR_WIN32_LEAN_AND_MEAN_ALREADY_DEFINED
#else
# 158 "../include/internal/ethread.h"
#  define WIN32_LEAN_AND_MEAN
#endif
# 160 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include <windows.h>
#endif /* expanded by -frewrite-includes */
# 160 "../include/internal/ethread.h"
# 161 "../include/internal/ethread.h"
#ifndef ETHR_WIN32_LEAN_AND_MEAN_ALREADY_DEFINED
#  undef WIN32_LEAN_AND_MEAN
#endif
# 164 "../include/internal/ethread.h"

#if defined(_MSC_VER)

#if ETHR_SIZEOF_LONG == 4
#define ETHR_HAVE_INT32_T 1
typedef long ethr_sint32_t;
typedef unsigned long ethr_uint32_t;
#endif
# 172 "../include/internal/ethread.h"

#if ETHR_SIZEOF___INT64 == 8
#define ETHR_HAVE_INT64_T 1
typedef __int64 ethr_sint64_t;
typedef unsigned __int64 ethr_uint64_t;
#endif
# 178 "../include/internal/ethread.h"

#endif
# 180 "../include/internal/ethread.h"

struct ethr_join_data_;

/* Types */
typedef struct {
    long id;
    struct ethr_join_data_ *jdata;
} ethr_tid; /* thread id type */

typedef DWORD ethr_tsd_key;

#undef ETHR_HAVE_ETHR_SIG_FUNCS

#define ETHR_USE_OWN_RWMTX_IMPL__

#define ETHR_YIELD() (Sleep(0), 0)

#elif defined(ETHR_OSE_THREADS)
# 198 "../include/internal/ethread.h"

#if 0 /* expanded by -frewrite-includes */
#include "ose.h"
#endif /* expanded by -frewrite-includes */
# 199 "../include/internal/ethread.h"
# 200 "../include/internal/ethread.h"
#undef NIL

#if defined(ETHR_HAVE_PTHREAD_H)
#if 0 /* expanded by -frewrite-includes */
#include <pthread.h>
#endif /* expanded by -frewrite-includes */
# 203 "../include/internal/ethread.h"
# 204 "../include/internal/ethread.h"
#endif
# 205 "../include/internal/ethread.h"

typedef struct {
  PROCESS id;
  unsigned int tsd_key_index;
  void *res;
} ethr_tid;

typedef OSPPDKEY ethr_tsd_key;

#undef ETHR_HAVE_ETHR_SIG_FUNCS

/* Out own RW mutexes are probably faster, but use OSEs mutexes */
#define ETHR_USE_OWN_RWMTX_IMPL__

#else /* No supported thread lib found */
# 220 "../include/internal/ethread.h"

#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL
#define ETHR_NO_THREAD_LIB
#else
# 224 "../include/internal/ethread.h"
#error "No supported thread lib found"
#endif
# 226 "../include/internal/ethread.h"

#endif
# 228 "../include/internal/ethread.h"

#ifndef ETHR_HAVE_INT32_T
#if ETHR_SIZEOF_INT == 4
#define ETHR_HAVE_INT32_T 1
typedef int ethr_sint32_t;
typedef unsigned int ethr_uint32_t;
#elif ETHR_SIZEOF_LONG == 4
# 235 "../include/internal/ethread.h"
#define ETHR_HAVE_INT32_T 1
typedef long ethr_sint32_t;
typedef unsigned long ethr_uint32_t;
#endif
# 239 "../include/internal/ethread.h"
#endif
# 240 "../include/internal/ethread.h"

#ifndef ETHR_HAVE_INT64_T
#if ETHR_SIZEOF_INT == 8
#define ETHR_HAVE_INT64_T 1
typedef int ethr_sint64_t;
typedef unsigned int ethr_uint64_t;
#elif ETHR_SIZEOF_LONG == 8
# 247 "../include/internal/ethread.h"
#define ETHR_HAVE_INT64_T 1
typedef long ethr_sint64_t;
typedef unsigned long ethr_uint64_t;
#elif ETHR_SIZEOF_LONG_LONG == 8
# 251 "../include/internal/ethread.h"
#define ETHR_HAVE_INT64_T 1
typedef long long ethr_sint64_t;
typedef unsigned long long ethr_uint64_t;
#endif
# 255 "../include/internal/ethread.h"
#endif
# 256 "../include/internal/ethread.h"

#if ETHR_SIZEOF_PTR == 4
#ifndef ETHR_HAVE_INT32_T
#error "No 32-bit integer type found"
#endif
# 261 "../include/internal/ethread.h"
typedef ethr_sint32_t ethr_sint_t;
typedef ethr_uint32_t ethr_uint_t;
#elif ETHR_SIZEOF_PTR == 8
# 264 "../include/internal/ethread.h"
#ifndef ETHR_HAVE_INT64_T
#error "No 64-bit integer type found"
#endif
# 267 "../include/internal/ethread.h"
typedef ethr_sint64_t ethr_sint_t;
typedef ethr_uint64_t ethr_uint_t;
#endif
# 270 "../include/internal/ethread.h"

#if defined(ETHR_SIZEOF___INT128_T) && ETHR_SIZEOF___INT128_T == 16
#define ETHR_HAVE_INT128_T
typedef __int128_t ethr_sint128_t;
typedef __uint128_t ethr_uint128_t;
#endif
# 276 "../include/internal/ethread.h"

#define ETHR_FATAL_ERROR__(ERR) \
  ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR))

ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file,
					 int line,
					 const char *func,
					 int err);

#if !ETHR_AT_LEAST_GCC_VSN__(2, 96, 0)
#define __builtin_expect(X, Y) (X)
#endif
# 288 "../include/internal/ethread.h"

#if ETHR_AT_LEAST_GCC_VSN__(3, 1, 1)
#  define ETHR_CHOOSE_EXPR __builtin_choose_expr
#else
# 292 "../include/internal/ethread.h"
#  define ETHR_CHOOSE_EXPR(B, E1, E2) ((B) ? (E1) : (E2))
#endif
# 294 "../include/internal/ethread.h"

#if ((defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) \
     || (defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_AMD64))))
#  define ETHR_X86_RUNTIME_CONF__

#  define ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ \
  (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg != 0, 1))
#  define ETHR_X86_RUNTIME_CONF_HAVE_NO_DW_CMPXCHG__ \
  (__builtin_expect(ethr_runtime__.conf.have_dw_cmpxchg == 0, 0))
#  define ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ \
  (__builtin_expect(ethr_runtime__.conf.have_sse2 != 0, 1))
#  define ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__ \
  (__builtin_expect(ethr_runtime__.conf.have_sse2 == 0, 0))
#endif
# 308 "../include/internal/ethread.h"

#if (defined(__GNUC__) \
     && !defined(ETHR_PPC_HAVE_LWSYNC) \
     && !defined(ETHR_PPC_HAVE_NO_LWSYNC) \
     && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)))
#  define ETHR_PPC_RUNTIME_CONF__

#  define ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ \
  (__builtin_expect(ethr_runtime__.conf.have_lwsync != 0, 1))  
#  define ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__ \
  (__builtin_expect(ethr_runtime__.conf.have_lwsync == 0, 0))
#endif
# 320 "../include/internal/ethread.h"

typedef struct {
#if defined(ETHR_X86_RUNTIME_CONF__)
    int have_dw_cmpxchg;
    int have_sse2;
#endif
# 326 "../include/internal/ethread.h"
#if defined(ETHR_PPC_RUNTIME_CONF__)
    int have_lwsync;
#endif
# 329 "../include/internal/ethread.h"
    int dummy;
} ethr_runtime_conf_t;


typedef union {
    ethr_runtime_conf_t conf;
    char pad__[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_runtime_conf_t))+ETHR_CACHE_LINE_SIZE];
} ethr_runtime_t;


extern ethr_runtime_t ethr_runtime__;

/* For native CPU-optimised atomics, spinlocks, and rwlocks. */
#if !defined(ETHR_DISABLE_NATIVE_IMPLS)
#  if defined(__GNUC__)
#    if defined(ETHR_PREFER_GCC_NATIVE_IMPLS)
#if 0 /* expanded by -frewrite-includes */
#      include "gcc/ethread.h"
#endif /* expanded by -frewrite-includes */
# 345 "../include/internal/ethread.h"
# 346 "../include/internal/ethread.h"
#    elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
# 347 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#      include "libatomic_ops/ethread.h"
#endif /* expanded by -frewrite-includes */
# 347 "../include/internal/ethread.h"
# 348 "../include/internal/ethread.h"
#    endif
# 349 "../include/internal/ethread.h"
#    if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
#      if ETHR_SIZEOF_PTR == 4
#        if defined(__i386__)
#if 0 /* expanded by -frewrite-includes */
#          include "i386/ethread.h"
#endif /* expanded by -frewrite-includes */
# 352 "../include/internal/ethread.h"
# 353 "../include/internal/ethread.h"
#        elif (defined(__powerpc__)||defined(__ppc__))&&!defined(__powerpc64__)
# 354 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#          include "ppc32/ethread.h"
#endif /* expanded by -frewrite-includes */
# 354 "../include/internal/ethread.h"
# 355 "../include/internal/ethread.h"
#        elif defined(__sparc__)
# 356 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#          include "sparc32/ethread.h"
#endif /* expanded by -frewrite-includes */
# 356 "../include/internal/ethread.h"
# 357 "../include/internal/ethread.h"
#        elif defined(__tile__)
# 358 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#          include "tile/ethread.h"
#endif /* expanded by -frewrite-includes */
# 358 "../include/internal/ethread.h"
# 359 "../include/internal/ethread.h"
#        endif
# 360 "../include/internal/ethread.h"
#      elif ETHR_SIZEOF_PTR == 8
# 361 "../include/internal/ethread.h"
#        if defined(__x86_64__)
#if 0 /* expanded by -frewrite-includes */
#          include "x86_64/ethread.h"
#endif /* expanded by -frewrite-includes */
# 362 "../include/internal/ethread.h"
# 363 "../include/internal/ethread.h"
#        elif defined(__sparc__) && defined(__arch64__)
# 364 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#          include "sparc64/ethread.h"
#endif /* expanded by -frewrite-includes */
# 364 "../include/internal/ethread.h"
# 365 "../include/internal/ethread.h"
#        endif
# 366 "../include/internal/ethread.h"
#      endif
# 367 "../include/internal/ethread.h"
#      if ETHR_HAVE_GCC___ATOMIC_BUILTINS
#if 0 /* expanded by -frewrite-includes */
#        include "gcc/ethread.h"
#endif /* expanded by -frewrite-includes */
# 368 "../include/internal/ethread.h"
# 1 "../include/internal/gcc/ethread.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010-2015. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Native atomic ethread support when using gcc's __atomic
 *              and __sync builtins
 * Author: Rickard Green
 */

#if !defined(ETHREAD_GCC_NATIVE_H__) && ETHR_GCC_COMPILER
#define ETHREAD_GCC_NATIVE_H__

#ifndef ETHR_MEMBAR
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_membar.h"
#endif /* expanded by -frewrite-includes */
# 31 "../include/internal/gcc/ethread.h"
# 1 "../include/internal/gcc/ethr_membar.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2015. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Memory barriers when using gcc's __atomic and
 *              __sync builtins
 * Author: Rickard Green
 *
 * Note: The C11 memory model implemented by gcc's __atomic
 *       builtins does not match the ethread API very well.
 *
 *       A function with a barrier postfix in the ethread atomic
 *       API needs to ensure that all stores and loads are
 *       ordered around it according to the semantics of the
 *       barrier specified.
 *
 *       The C11 aproch is different. The __atomic builtins
 *       API takes a memory model parameter. Assuming that all
 *       memory syncronizations using the involved atomic
 *       variables are made using this API, the synchronizations
 *       will adhere to the memory models used. That is, you do
 *       *not* know how loads and stores will be ordered around
 *       a specific __atomic operation in the general case. You
 *       only know the total effect of the combination of
 *       operations issued will adhere to the model.
 *
 *       This limits how we can use the __atomic builtins. What
 *       we cannot use:
 *
 *       1. We cannot rely on __atomic_thread_fence() to issue
 *          any specific memory barriers at all. This regardless
 *          of memory model parameter passed. That is, we cannot
 *          use the __atomic_thread_fence() builtin at all.
 *
 *          Why is this? If all __atomic builtins accessing
 *          memory issue memory barriers, __atomic_thread_fence()
 *          does not have to issue memory barriers. The
 *          implementation for the Itanium architecture is an
 *          example of this. Even using the __ATOMIC_RELAXED
 *          memory model all __atomic builtins accessing memory
 *          will issue memory barriers. Due to this no memory
 *          barriers at all will be issued by
 *           __atomic_thread_fence() using either one of the
 *          __ATOMIC_CONSUME, __ATOMIC_ACQUIRE, or
 *          __ATOMIC_RELEASE memory models.
 *
 *       2. We cannot rely on any __atomic builtin with the
 *          __ATOMIC_SEQ_CST memory model parameters to
 *          issue any specific memory barriers. That is, we
 *          cannot use these memory models at all.
 *
 *          Why is this? Since all synchronizations is expected
 *          to be made using the __atomic builtins, memory
 *          barriers only have to be issued by some of them,
 *          and you do not know which ones wont issue memory
 *          barriers.
 *
 *          One can easily be fooled into believing that when
 *          using the __ATOMIC_SEQ_CST memory model on all
 *          operations, all operations will issue full memory
 *          barriers. This is however not the case. The
 *          implementation for the x86_64 architecture is an
 *          example of this. Since all operations except loads
 *          issue full memory barriers, no memory barriers at
 *          all is issued by loads. This could also be
 *          implemented by issuing a full memory barrier on
 *          loads, but no barrier at all on stores.
 *
 *       What can be used then?
 *       1. All (legacy) __sync builtins implying full memory
 *          barriers issued.
 *       2. All __atomic builtins using the __ATOMIC_RELAXED
 *          memory model can, of course, be used. This since
 *          no ordering guarantees at all are made.
 *       3. All __atomic builtins accessing memory using the
 *          __ATOMIC_ACQUIRE and __ATOMIC_RELEASE memory
 *          models. This since an __atomic builtin memory
 *          access using the __ATOMIC_ACQUIRE must at least
 *          issue an aquire memory barrier and an __atomic
 *          builtin memory acess with the __ATOMIC_RELEASE
 *          memory model must at least issue a release memory
 *          barrier. Otherwise the two can not be paired.
 *       4. All __atomic builtins accessing memory using the
 *          __ATOMIC_CONSUME builtin can be used for the same
 *          reason __ATOMIC_ACQUIRE can be used. The ethread
 *          atomic framework implementing the ethread API
 *          using native implementations does not expect the
 *          native implementations to produce versions with
 *          data dependent read barriers, so until the
 *          framework is changed we haven't got any use for
 *          for it.
 *
 *       For some architectures we have our own memory barrier
 *       implementations. We prefer to use these since they
 *       should be as fine grained as possible. For other
 *       architectures we use the __sync_synchronize() builtin
 *       which issue a full memory barrier. For these
 *       architectures we have to assume that all loads and
 *       stores can be reordered without limitation. That is,
 *       unnecessary memory barriers will be issued if such
 *       reordering actually cannot occur.
 */

/*
 * We prefer to use our own memory barrier implementation if
 * such exist instead of using __sync_synchronize()...
 */
#if defined(__i386__) || defined(__x86_64__)
#if 0 /* expanded by -frewrite-includes */
#  include "../i386/ethr_membar.h"
#endif /* expanded by -frewrite-includes */
# 126 "../include/internal/gcc/ethr_membar.h"
# 127 "../include/internal/gcc/ethr_membar.h"
#elif defined(__sparc__)
# 128 "../include/internal/gcc/ethr_membar.h"
#if 0 /* expanded by -frewrite-includes */
#  include "../sparc32/ethr_membar.h"
#endif /* expanded by -frewrite-includes */
# 128 "../include/internal/gcc/ethr_membar.h"
# 129 "../include/internal/gcc/ethr_membar.h"
#elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
# 130 "../include/internal/gcc/ethr_membar.h"
#if 0 /* expanded by -frewrite-includes */
#  include "../ppc32/ethr_membar.h"
#endif /* expanded by -frewrite-includes */
# 130 "../include/internal/gcc/ethr_membar.h"
# 131 "../include/internal/gcc/ethr_membar.h"
#elif !defined(ETHR_GCC_ATOMIC_MEMBAR_H__)			\
    && (ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION			\
	|| ETHR_HAVE___sync_synchronize				\
	|| (ETHR_HAVE___sync_val_compare_and_swap & 12))
# 135 "../include/internal/gcc/ethr_membar.h"
#define ETHR_GCC_ATOMIC_MEMBAR_H__

#define ETHR_LoadLoad	(1 << 0)
#define ETHR_LoadStore	(1 << 1)
#define ETHR_StoreLoad	(1 << 2)
#define ETHR_StoreStore	(1 << 3)

#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")

#if ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION

static __inline__ __attribute__((__always_inline__)) void
ethr_full_fence__(void)
{
    __asm__ __volatile__("dmb sy" : : : "memory");
}

static __inline__ __attribute__((__always_inline__)) void
ethr_store_fence__(void)
{
    __asm__ __volatile__("dmb st" : : : "memory");
}

#define ETHR_MEMBAR(B) \
 ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, ethr_store_fence__(), ethr_full_fence__())

#elif ETHR_HAVE___sync_synchronize
# 162 "../include/internal/gcc/ethr_membar.h"

static __inline__ __attribute__((__always_inline__)) void
ethr_full_fence__(void)
{
    /*
     * The compiler barriers are here to fix missing clobbers
     * in __sync_synchronize() when using buggy LLVM
     * implementation of __sync_synchronize(). They
     * do not introduce any unnecessary overhead when used
     * here, so we use them for all systems.
     */
    ETHR_COMPILER_BARRIER;
    __sync_synchronize();
    ETHR_COMPILER_BARRIER;
}

#else /* !ETHR_HAVE___sync_synchronize */
# 179 "../include/internal/gcc/ethr_membar.h"

/*
 * Buggy __sync_synchronize(); call __sync_val_compare_and_swap()
 * instead which imply a full memory barrier (and hope that one
 * isn't buggy too).
 */

#if (ETHR_HAVE___sync_val_compare_and_swap & 4)
#  define ETHR_MB_T__ ethr_sint32_t
#elif (ETHR_HAVE___sync_val_compare_and_swap & 8)
# 189 "../include/internal/gcc/ethr_membar.h"
#  define ETHR_MB_T__ ethr_sint64_t
#endif
# 191 "../include/internal/gcc/ethr_membar.h"

static __inline__ __attribute__((__always_inline__)) void
ethr_full_fence__(void)
{
    volatile ETHR_MB_T__ x = 0;
    (void) __sync_val_compare_and_swap(&x, (ETHR_MB_T__) 0, (ETHR_MB_T__) 1);
}

#endif /* !ETHR_HAVE___sync_synchronize */
# 200 "../include/internal/gcc/ethr_membar.h"

#ifndef ETHR_MEMBAR
#  define ETHR_MEMBAR(B) ethr_full_fence__()
#endif
# 204 "../include/internal/gcc/ethr_membar.h"

/*
 * Define ETHR_READ_DEPEND_MEMORY_BARRIER for all architechtures
 * not known to order data dependent loads
 */

#if !defined(__ia64__) && !defined(__arm__)
#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
#endif
# 213 "../include/internal/gcc/ethr_membar.h"

#endif /* ETHR_GCC_ATOMIC_MEMBAR_H__ */
# 215 "../include/internal/gcc/ethr_membar.h"
# 32 "../include/internal/gcc/ethread.h" 2
#endif
# 33 "../include/internal/gcc/ethread.h"

#define ETHR_GCC_VERSIONS_MASK__ 28

#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__
#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#undef ETHR_GCC_RELAXED_VERSIONS__
#undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#undef ETHR_GCC_ACQB_VERSIONS__
#undef ETHR_GCC_ACQB_MOD_VERSIONS__
#undef ETHR_GCC_RELB_VERSIONS__
#undef ETHR_GCC_RELB_MOD_VERSIONS__
#undef ETHR_GCC_MB_MOD_VERSIONS__

/*
 * True GNU GCCs before version 4.8 do not emit a memory barrier
 * after the load in the __atomic_load_n(_, __ATOMIC_ACQUIRE)
 * case (which is needed on most architectures).
 */
#undef ETHR___atomic_load_ACQUIRE_barrier_bug
#if ETHR_GCC_COMPILER != ETHR_GCC_COMPILER_TRUE
/*
 * A gcc compatible compiler. We have no information
 * about the existence of this bug, but we assume
 * that it is not impossible that it could have
 * been "inherited". Therefore, until we are certain
 * that the bug does not exist, we assume that it
 * does.
 */
#  define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__
#elif !ETHR_AT_LEAST_GCC_VSN__(4, 8, 0)
# 65 "../include/internal/gcc/ethread.h"
/* True gcc of version < 4.8, i.e., bug exist... */
#  define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__
#else /* True gcc of version >= 4.8 */
# 68 "../include/internal/gcc/ethread.h"
/*
 * Sizes less than or equal to word size have been fixed,
 * but double word size has not been fixed.
 */
#  if ETHR_SIZEOF_PTR == 8
#    define ETHR___atomic_load_ACQUIRE_barrier_bug \
    (~(8|4) & ETHR_GCC_VERSIONS_MASK__)
#  elif ETHR_SIZEOF_PTR == 4
# 76 "../include/internal/gcc/ethread.h"
#    define ETHR___atomic_load_ACQUIRE_barrier_bug \
    (~4 & ETHR_GCC_VERSIONS_MASK__)
#  else
# 79 "../include/internal/gcc/ethread.h"
#    error word size not supported
#  endif
# 81 "../include/internal/gcc/ethread.h"
#endif
# 82 "../include/internal/gcc/ethread.h"

#define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ 0
#define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 0
#define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 0
#define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 0
#define ETHR_GCC_RELAXED_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#define ETHR_GCC_RELAXED_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__

#if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS
#  define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#else
# 96 "../include/internal/gcc/ethread.h"
/*
 * This is currently the default (on most platforms) since
 * we've seen too many memory barrier bugs produced by gcc...
 */
#  define ETHR_GCC_ACQB_VERSIONS__ 0
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ 0
#  define ETHR_GCC_RELB_VERSIONS__ 0
#  define ETHR_GCC_RELB_MOD_VERSIONS__ 0
#endif
# 105 "../include/internal/gcc/ethread.h"
/*
 * In the general case we do not want any full barrier versions
 * if we can implement more relaxed ones (using __atomic_* builtins).
 * This since the implementations normally need extra memory barrier
 * instructions to implement these. The x86/x86_64 implementations
 * are an exception see below.
 */
#define ETHR_GCC_MB_MOD_VERSIONS__ \
    (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___atomic_compare_exchange_n)

#if ETHR_SIZEOF_PTR == 8
#  define ETHR_GCC_VOLATILE_BIT_MASK__ 12
#elif ETHR_SIZEOF_PTR == 4
# 118 "../include/internal/gcc/ethread.h"
#  define ETHR_GCC_VOLATILE_BIT_MASK__ 4
#endif
# 120 "../include/internal/gcc/ethread.h"

#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__)	\
    || defined(__powerpc__) || defined(__ppc__) || defined(__mips__)	\
    || defined(__alpha__) || defined(__ia64__)

/*
 * Aligned volatile stores and loads of data smaller
 * than or equal to word size are atomic...
 */
#  undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__
#  define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ ETHR_GCC_VOLATILE_BIT_MASK__
#  undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#  define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ ETHR_GCC_VOLATILE_BIT_MASK__

#elif defined(__arm__)
# 135 "../include/internal/gcc/ethread.h"

/* volatile stores are problematic on some machines... */
#  undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#  define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ ETHR_GCC_VOLATILE_BIT_MASK__

#endif
# 141 "../include/internal/gcc/ethread.h"

#if defined(__ia64__)

/* Volatile stores produce stores with release barriers. */
#  undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#  define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ ETHR_GCC_VOLATILE_BIT_MASK__

/* Volatile loads produce loads with acquire barrier. */
#  undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#  define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ ETHR_GCC_VOLATILE_BIT_MASK__

/*
 * We trust gcc to produce acquire/release barriers on itanium.
 * Since all atomic ops also have at least acquire or release
 * barriers (also when passed the relaxed memory model) it
 * would be very inefficient not to use these as native
 * barriers on Itanium.
 */
#  undef ETHR_GCC_ACQB_VERSIONS__
#  define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  undef ETHR_GCC_ACQB_MOD_VERSIONS__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  undef ETHR_GCC_RELB_VERSIONS__
#  define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  undef ETHR_GCC_RELB_MOD_VERSIONS__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__

/*
 * Itanium is not effected by the load acquire
 * bug since the barrier is part of the instruction
 * on Itanium (ld.acq), and not a separate instruction
 * as on most platforms.
 */
#  undef ETHR___atomic_load_ACQUIRE_barrier_bug
#  define ETHR___atomic_load_ACQUIRE_barrier_bug 0

/*
 * No point exposing relaxed versions since they are
 * implemended using either acquire or release
 * barriers.
 */
#  undef ETHR_GCC_RELAXED_VERSIONS__
#  define ETHR_GCC_RELAXED_VERSIONS__ 0

/* #endif defined(__ia64__) */
#elif defined(__i386__) || defined(__x86_64__)
# 187 "../include/internal/gcc/ethread.h"

/*
 * Want full barrier versions of all modification
 * operations since all of these are implemented
 * using locked instructions implying full memory
 * barriers.
 */
#  undef ETHR_GCC_MB_MOD_VERSIONS__
#  define ETHR_GCC_MB_MOD_VERSIONS__ ETHR_HAVE___sync_val_compare_and_swap

/*
 * No point exposing acquire/release versions
 * when we got full memory barrier versions
 * of modification operations since all of these
 * are implemented using locked instructions
 * implying full memory barriers.
 */
#  if ETHR_GCC_ACQB_MOD_VERSIONS__
#    undef ETHR_GCC_ACQB_MOD_VERSIONS__
#    define ETHR_GCC_ACQB_MOD_VERSIONS__ \
    (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___sync_val_compare_and_swap)
#  endif
# 209 "../include/internal/gcc/ethread.h"
#  if ETHR_GCC_RELB_MOD_VERSIONS__
#    undef ETHR_GCC_RELB_MOD_VERSIONS__
#    define ETHR_GCC_RELB_MOD_VERSIONS__ \
    (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___sync_val_compare_and_swap)
#  endif
# 214 "../include/internal/gcc/ethread.h"

#  ifdef ETHR_X86_OUT_OF_ORDER

/* See above... */
#    undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#    define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0

#  else /* !ETHR_X86_OUT_OF_ORDER, i.e., we don't use any x86-OOO instructions... */
# 222 "../include/internal/gcc/ethread.h"

/*
 * Not effected by the load acquire barrier bug,
 * since no barrier at all is needed for a load
 * acquire...
 */
#    undef ETHR___atomic_load_ACQUIRE_barrier_bug
#    define ETHR___atomic_load_ACQUIRE_barrier_bug 0

/* Stores imply release barriers semantics. */
#    undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#    define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ ETHR_GCC_VOLATILE_BIT_MASK__

/* Loads imply acquire barrier semantics. */
#    undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#    define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ ETHR_GCC_VOLATILE_BIT_MASK__

/*
 * Trust load acquire and store release for sizes
 * where volatile operation implies these barrier
 * semantics since no barriers are needed.
 */
#    if !ETHR_GCC_ACQB_VERSIONS__
#      undef ETHR_GCC_ACQB_VERSIONS__
#      define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VOLATILE_BIT_MASK__
#    endif
# 248 "../include/internal/gcc/ethread.h"
#    if !ETHR_GCC_RELB_VERSIONS__
#      undef ETHR_GCC_RELB_VERSIONS__
#      define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VOLATILE_BIT_MASK__
#    endif
# 252 "../include/internal/gcc/ethread.h"

/*
 * No point exposing relaxed versions at all since
 * all mod operations are implemented with locked
 * instructions implying full memory barriers and
 * volatile store and load imply release and
 * acquire barrier semantics.
 */
#    undef ETHR_GCC_RELAXED_VERSIONS__
#    define ETHR_GCC_RELAXED_VERSIONS__ 0

#  endif /* !ETHR_X86_OUT_OF_ORDER */
# 264 "../include/internal/gcc/ethread.h"

/* #endif defined(__i386__) || defined(__x86_64__) */
#elif defined(__powerpc__) || defined(__ppc__)
# 267 "../include/internal/gcc/ethread.h"

#  if !defined(ETHR_PPC_HAVE_LWSYNC)
/*
 * Release barriers are typically implemented using
 * the lwsync instruction. We want our runtime
 * configure test to determine if the lwsync
 * instruction is available on the system or not
 * before we use it. Therefore, do not implement any
 * native ops using the __ATOMIC_RELEASE model.
 */
#    undef ETHR_GCC_RELB_VERSIONS__
#    define ETHR_GCC_RELB_VERSIONS__ 0
#    if defined(ETHR_GCC_IMPLEMENT_ACQB_USING_LWSYNC)
/*
 * Acquire barriers are usually implemented by other means
 * than lwsync, but can be implemented using lwsync. Define
 * ETHR_GCC_IMPLEMENT_ACQB_USING_LWSYNC if acquire barriers
 * are implemented using lwsync.
 */
#      undef ETHR_GCC_ACQB_VERSIONS__
#      define ETHR_GCC_ACQB_VERSIONS__ 0
#    endif
# 289 "../include/internal/gcc/ethread.h"
#  endif
# 290 "../include/internal/gcc/ethread.h"

#endif /* defined(__powerpc__) || defined(__ppc__) */
# 292 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELAXED_VERSIONS__
#  undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#  define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0
#endif
# 297 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_ACQB_VERSIONS__
#  undef ETHR_GCC_ACQB_MOD_VERSIONS__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ 0
#endif
# 302 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELB_VERSIONS__
#  undef ETHR_GCC_RELB_MOD_VERSIONS__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ 0
#endif
# 307 "../include/internal/gcc/ethread.h"

#if !defined(ETHR_HAVE_NATIVE_ATOMIC32)
#  define ETHR_ATOMIC_WANT_32BIT_IMPL__
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_atomic.h"
#endif /* expanded by -frewrite-includes */
# 310 "../include/internal/gcc/ethread.h"
# 1 "../include/internal/gcc/ethr_atomic.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010-2015. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Native atomics ethread support using gcc's __atomic
 *              and __sync builtins
 * Author: Rickard Green
 *
 * Note: The C11 memory model implemented by gcc's __atomic
 *       builtins does not match the ethread API very well.
 *
 *       Due to this we cannot use the __ATOMIC_SEQ_CST
 *       memory model. For more information see the comment
 *       in the begining of ethr_membar.h in this directory.
 */

#undef ETHR_INCLUDE_ATOMIC_IMPL__
#if !defined(ETHR_GCC_ATOMIC_ATOMIC32_H__)		\
    && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)		\
    && ((ETHR_HAVE___sync_val_compare_and_swap & 4)	\
	|| (ETHR_HAVE___atomic_compare_exchange_n & 4))

#define ETHR_GCC_ATOMIC_ATOMIC32_H__
#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
#undef ETHR_ATOMIC_WANT_32BIT_IMPL__

#elif !defined(ETHR_GCC_ATOMIC64_H__)			\
    && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)		\
    && ((ETHR_HAVE___sync_val_compare_and_swap & 8)	\
	|| (ETHR_HAVE___atomic_compare_exchange_n & 8))
# 48 "../include/internal/gcc/ethr_atomic.h"

#define ETHR_GCC_ATOMIC64_H__
#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
#undef ETHR_ATOMIC_WANT_64BIT_IMPL__

#endif
# 54 "../include/internal/gcc/ethr_atomic.h"

#ifdef ETHR_INCLUDE_ATOMIC_IMPL__

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#define ETHR_HAVE_NATIVE_ATOMIC32 1
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic32_t
#define ETHR_AINT_T__ ethr_sint32_t
#if ((ETHR_HAVE___sync_val_compare_and_swap & 4) \
     && (ETHR_HAVE___atomic_compare_exchange_n & 4))
#  define ETHR_NATIVE_ATOMIC32_IMPL "gcc_atomic_and_sync_builtins"
#elif (ETHR_HAVE___atomic_compare_exchange_n & 4)
# 66 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_NATIVE_ATOMIC32_IMPL "gcc_atomic_builtins"
#elif (ETHR_HAVE___sync_val_compare_and_swap & 4)
# 68 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_NATIVE_ATOMIC32_IMPL "gcc_sync_builtins"
#else
# 70 "../include/internal/gcc/ethr_atomic.h"
#  error "!?"
#endif
# 72 "../include/internal/gcc/ethr_atomic.h"
#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
# 73 "../include/internal/gcc/ethr_atomic.h"
#define ETHR_HAVE_NATIVE_ATOMIC64 1
#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#define ETHR_ATMC_T__ ethr_native_atomic64_t
#define ETHR_AINT_T__ ethr_sint64_t
#if ((ETHR_HAVE___sync_val_compare_and_swap & 8) \
     && (ETHR_HAVE___atomic_compare_exchange_n & 8))
#  define ETHR_NATIVE_ATOMIC64_IMPL "gcc_atomic_and_sync_builtins"
#elif (ETHR_HAVE___atomic_compare_exchange_n & 8)
# 81 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_NATIVE_ATOMIC64_IMPL "gcc_atomic_builtins"
#elif (ETHR_HAVE___sync_val_compare_and_swap & 8)
# 83 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_NATIVE_ATOMIC64_IMPL "gcc_sync_builtins"
#else
# 85 "../include/internal/gcc/ethr_atomic.h"
#  error "!?"
#endif
# 87 "../include/internal/gcc/ethr_atomic.h"
#else
# 88 "../include/internal/gcc/ethr_atomic.h"
#error "Unsupported integer size"
#endif
# 90 "../include/internal/gcc/ethr_atomic.h"

#undef ETHR_NATIVE_ATOMIC_IMPL__

typedef struct {
    volatile ETHR_AINT_T__ value;
} ETHR_ATMC_T__;

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
#else
# 102 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
#endif
# 104 "../include/internal/gcc/ethr_atomic.h"


static ETHR_INLINE ETHR_AINT_T__ *
ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
{
    return (ETHR_AINT_T__ *) &var->value;
}

/*
 * set()
 */
#if (ETHR_HAVE___atomic_store_n & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELAXED_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
#else
# 122 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
#endif
# 124 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE void
ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
{
    __atomic_store_n(&var->value, value, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_VERSIONS__ */
# 132 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_RELB_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
#else
# 138 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
#endif
# 140 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
{
    __atomic_store_n(&var->value, value, __ATOMIC_RELEASE);
}

#endif /* ETHR_GCC_RELB_VERSIONS__ */
# 148 "../include/internal/gcc/ethr_atomic.h"

#elif (ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ & ETHR_INCLUDE_ATOMIC_IMPL__)
# 150 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_RELAXED_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
#else
# 156 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
#endif
# 158 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE void
ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
{
    var->value = value;
}

#endif /* ETHR_GCC_RELAXED_VERSIONS__ */
# 166 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELB_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
#else
# 174 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
#endif
# 176 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE void
ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
{
    var->value = value;
}

#endif /* ETHR_GCC_RELB_VERSIONS__ */
# 184 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ */
# 186 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ */
# 188 "../include/internal/gcc/ethr_atomic.h"

/*
 * read()
 */

#if (ETHR_HAVE___atomic_load_n & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELAXED_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
#else
# 200 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
#endif
# 202 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
{
    return __atomic_load_n(&var->value, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_VERSIONS__ */
# 210 "../include/internal/gcc/ethr_atomic.h"

#if ((ETHR_GCC_ACQB_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__) \
     & ~ETHR___atomic_load_ACQUIRE_barrier_bug)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
#else
# 217 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
#endif
# 219 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
{
    return __atomic_load_n(&var->value, __ATOMIC_ACQUIRE);
}

#endif /* ETHR_GCC_ACQB_VERSIONS__ */
# 227 "../include/internal/gcc/ethr_atomic.h"

#elif (ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ & ETHR_INCLUDE_ATOMIC_IMPL__)
# 229 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_RELAXED_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
#else
# 235 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
#endif
# 237 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
{
    return var->value;
}

#endif /* ETHR_GCC_RELAXED_VERSIONS__ */
# 245 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_ACQB_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
#else
# 253 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
#endif
# 255 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
{
    return var->value;
}

#endif /* ETHR_GCC_ACQB_VERSIONS__ */
# 263 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ */
# 265 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ */
# 267 "../include/internal/gcc/ethr_atomic.h"

/*
 * add_return()
 */
#if (ETHR_HAVE___atomic_add_fetch & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELAXED_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
#else
# 278 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1
#endif
# 280 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
    return __atomic_add_fetch(&var->value, incr, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_MOD_VERSIONS__ */
# 288 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_ACQB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
#else
# 294 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB 1
#endif
# 296 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
    return __atomic_add_fetch(&var->value, incr, __ATOMIC_ACQUIRE);
}

#endif /* ETHR_GCC_ACQB_MOD_VERSIONS__ */
# 304 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_RELB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB 1
#else
# 310 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB 1
#endif
# 312 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
    return __atomic_add_fetch(&var->value, incr, __ATOMIC_RELEASE);
}

#endif /* ETHR_GCC_RELB_MOD_VERSIONS__ */
# 320 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_HAVE___atomic_add_fetch */
# 322 "../include/internal/gcc/ethr_atomic.h"

#if ((ETHR_HAVE___sync_add_and_fetch & ETHR_INCLUDE_ATOMIC_IMPL__) \
     & ETHR_GCC_MB_MOD_VERSIONS__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
#else
# 329 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
#endif
# 331 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
{
    return __sync_add_and_fetch(&var->value, incr);
}

#endif /* ETHR_HAVE___sync_add_and_fetch */
# 339 "../include/internal/gcc/ethr_atomic.h"

/*
 * and_retold()
 */
#if (ETHR_HAVE___atomic_fetch_and & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELAXED_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
#else
# 350 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD 1
#endif
# 352 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __atomic_fetch_and(&var->value, mask, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_MOD_VERSIONS__ */
# 360 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_ACQB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1
#else
# 366 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB 1
#endif
# 368 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(and_retold_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __atomic_fetch_and(&var->value, mask, __ATOMIC_ACQUIRE);
}

#endif /* ETHR_GCC_ACQB_MOD_VERSIONS__ */
# 376 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_RELB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB 1
#else
# 382 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB 1
#endif
# 384 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(and_retold_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __atomic_fetch_and(&var->value, mask, __ATOMIC_RELEASE);
}

#endif /* ETHR_GCC_RELB_MOD_VERSIONS__ */
# 392 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_HAVE___atomic_fetch_and */
# 394 "../include/internal/gcc/ethr_atomic.h"

#if ((ETHR_HAVE___sync_fetch_and_and & ETHR_INCLUDE_ATOMIC_IMPL__) \
     & ETHR_GCC_MB_MOD_VERSIONS__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
#else
# 401 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
#endif
# 403 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __sync_fetch_and_and(&var->value, mask);
}

#endif /* ETHR_HAVE___sync_fetch_and_and */
# 411 "../include/internal/gcc/ethr_atomic.h"

/*
 * or_retold()
 */
#if (ETHR_HAVE___atomic_fetch_or & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELAXED_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
#else
# 422 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD 1
#endif
# 424 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __atomic_fetch_or(&var->value, mask, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_MOD_VERSIONS__ */
# 432 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_ACQB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1
#else
# 438 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB 1
#endif
# 440 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(or_retold_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __atomic_fetch_or(&var->value, mask, __ATOMIC_ACQUIRE);
}

#endif /* ETHR_GCC_ACQB_MOD_VERSIONS__ */
# 448 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_RELB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB 1
#else
# 454 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB 1
#endif
# 456 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(or_retold_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return __atomic_fetch_or(&var->value, mask, __ATOMIC_RELEASE);
}

#endif /* ETHR_GCC_RELB_MOD_VERSIONS__ */
# 464 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_HAVE___atomic_fetch_or */
# 466 "../include/internal/gcc/ethr_atomic.h"

#if ((ETHR_HAVE___sync_fetch_and_or & ETHR_INCLUDE_ATOMIC_IMPL__) \
     & ETHR_GCC_MB_MOD_VERSIONS__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
#else
# 473 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
#endif
# 475 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
{
    return (ETHR_AINT_T__) __sync_fetch_and_or(&var->value, mask);
}

#endif /* ETHR_HAVE___sync_fetch_and_or */
# 483 "../include/internal/gcc/ethr_atomic.h"

/*
 * cmpxchg()
 */
#if (ETHR_HAVE___atomic_compare_exchange_n & ETHR_INCLUDE_ATOMIC_IMPL__)

#if (ETHR_GCC_RELAXED_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
#else
# 494 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
#endif
# 496 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
			   ETHR_AINT_T__ new,
			   ETHR_AINT_T__ exp)
{
    ETHR_AINT_T__ xchg = exp;
    if (__atomic_compare_exchange_n(&var->value,
				    &xchg,
				    new,
				    0, /* No spurious failures, please */
				    __ATOMIC_RELAXED,
				    __ATOMIC_RELAXED))
	return exp;
    return xchg;
}

#endif /* ETHR_GCC_RELAXED_MOD_VERSIONS__ */
# 514 "../include/internal/gcc/ethr_atomic.h"

#if (ETHR_GCC_ACQB_MOD_VERSIONS__ & ETHR_INCLUDE_ATOMIC_IMPL__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
#else
# 520 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
#endif
# 522 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
				ETHR_AINT_T__ new,
				ETHR_AINT_T__ exp)
{
    ETHR_AINT_T__ xchg = exp;
    if (__atomic_compare_exchange_n(&var->value,
				    &xchg,
				    new,
				    0, /* No spurious failures, please */
				    __ATOMIC_ACQUIRE,
				    __ATOMIC_ACQUIRE))
	return exp;
    return xchg;
}

#endif /* ETHR_GCC_ACQB_MOD_VERSIONS__ */
# 540 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_HAVE___atomic_compare_exchange_n */
# 542 "../include/internal/gcc/ethr_atomic.h"

#if ((ETHR_HAVE___sync_val_compare_and_swap & ETHR_INCLUDE_ATOMIC_IMPL__) \
     & ETHR_GCC_MB_MOD_VERSIONS__)

#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
#else
# 549 "../include/internal/gcc/ethr_atomic.h"
#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
#endif
# 551 "../include/internal/gcc/ethr_atomic.h"

static ETHR_INLINE ETHR_AINT_T__
ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
			      ETHR_AINT_T__ new,
			      ETHR_AINT_T__ old)
{
    return __sync_val_compare_and_swap(&var->value, old, new);
}

#endif /* ETHR_HAVE___sync_val_compare_and_swap */
# 561 "../include/internal/gcc/ethr_atomic.h"

#endif /* ETHR_TRY_INLINE_FUNCS */
# 563 "../include/internal/gcc/ethr_atomic.h"

#undef ETHR_NATMC_FUNC__
#undef ETHR_ATMC_T__
#undef ETHR_AINT_T__
#undef ETHR_AINT_SUFFIX__

#endif
# 570 "../include/internal/gcc/ethr_atomic.h"
# 311 "../include/internal/gcc/ethread.h" 2
#endif
# 312 "../include/internal/gcc/ethread.h"

#if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_atomic.h"
#endif /* expanded by -frewrite-includes */
# 315 "../include/internal/gcc/ethread.h"
# 316 "../include/internal/gcc/ethread.h"
#endif
# 317 "../include/internal/gcc/ethread.h"

#if defined(__x86_64__)
/*
 * No instructions available for native implementation
 * of these for dw-atomics...
 */
#  undef ETHR_GCC_RELAXED_VERSIONS__
#  define ETHR_GCC_RELAXED_VERSIONS__ 0
#  undef ETHR_GCC_ACQB_VERSIONS__
#  define ETHR_GCC_ACQB_VERSIONS__ 0
#  undef ETHR_GCC_RELB_VERSIONS__
#  define ETHR_GCC_RELB_VERSIONS__ 0
#endif
# 330 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELAXED_VERSIONS__
#  undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#  define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0
#endif
# 335 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_ACQB_VERSIONS__
#  undef ETHR_GCC_ACQB_MOD_VERSIONS__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ 0
#endif
# 340 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELB_VERSIONS__
#  undef ETHR_GCC_RELB_MOD_VERSIONS__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ 0
#endif
# 345 "../include/internal/gcc/ethread.h"

#if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \
     && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \
     && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128)))
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_dw_atomic.h"
#endif /* expanded by -frewrite-includes */
# 349 "../include/internal/gcc/ethread.h"
# 1 "../include/internal/gcc/ethr_dw_atomic.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2015. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Native double word atomics using gcc's __atomic
 *              and __sync builtins
 * Author: Rickard Green
 *
 * Note: The C11 memory model implemented by gcc's __atomic
 *       builtins does not match the ethread API very well.
 *
 *       Due to this we cannot use the __ATOMIC_SEQ_CST
 *       memory model. For more information see the comment
 *       in the begining of ethr_membar.h in this directory.
 */

#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__
#if !defined(ETHR_GCC_ATOMIC_DW_ATOMIC_H__)				\
    && ((ETHR_HAVE___sync_val_compare_and_swap & (2*ETHR_SIZEOF_PTR))	\
	|| (ETHR_HAVE___atomic_compare_exchange_n & (2*ETHR_SIZEOF_PTR)))
#  define ETHR_GCC_ATOMIC_DW_ATOMIC_H__
#  define ETHR_INCLUDE_DW_ATOMIC_IMPL__
#endif
# 41 "../include/internal/gcc/ethr_dw_atomic.h"

#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__
#  define ETHR_HAVE_NATIVE_SU_DW_ATOMIC

#if ((ETHR_HAVE___sync_val_compare_and_swap & (2*ETHR_SIZEOF_PTR))	\
     && (ETHR_HAVE___atomic_compare_exchange_n & (2*ETHR_SIZEOF_PTR)))
#  define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc_atomic_and_sync_builtins"
#elif (ETHR_HAVE___atomic_compare_exchange_n & (2*ETHR_SIZEOF_PTR))
# 49 "../include/internal/gcc/ethr_dw_atomic.h"
#  define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc_atomic_builtins"
#elif (ETHR_HAVE___sync_val_compare_and_swap & (2*ETHR_SIZEOF_PTR))
# 51 "../include/internal/gcc/ethr_dw_atomic.h"
#  define ETHR_NATIVE_DW_ATOMIC_IMPL "gcc_sync_builtins"
#else
# 53 "../include/internal/gcc/ethr_dw_atomic.h"
#  error "!?"
#endif
# 55 "../include/internal/gcc/ethr_dw_atomic.h"

#  if ETHR_SIZEOF_PTR == 4
#    define ETHR_DW_NATMC_ALIGN_MASK__ 0x7
#    define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t
#  elif ETHR_SIZEOF_PTR == 8
# 60 "../include/internal/gcc/ethr_dw_atomic.h"
#    define ETHR_DW_NATMC_ALIGN_MASK__ 0xf
#    define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t
#  endif
# 63 "../include/internal/gcc/ethr_dw_atomic.h"

typedef volatile ETHR_NATIVE_SU_DW_SINT_T * ethr_native_dw_ptr_t;

/*
 * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned
 * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is
 * not common, and at least some glibc malloc implementations
 * only 4 byte align in 32-bit mode.
 *
 * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte
 * aligned memory in 32-bit mode. A malloc implementation that does
 * not adhere to these alignment requirements is seriously broken,
 * and we wont bother trying to work around it.
 *
 * Since memory alignment may be off by one word we need to align at
 * runtime. We, therefore, need an extra word allocated.
 */
#define ETHR_DW_NATMC_MEM__(VAR) \
   (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__])
typedef union {
    volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint;
    volatile ethr_sint_t sint[3];
    volatile char c[ETHR_SIZEOF_PTR*3];
} ethr_native_dw_atomic_t;

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)

#  ifdef ETHR_DEBUG
#    define ETHR_DW_DBG_ALIGNED__(PTR) \
       ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0);
#  else
# 94 "../include/internal/gcc/ethr_dw_atomic.h"
#    define ETHR_DW_DBG_ALIGNED__(PTR)
#  endif
# 96 "../include/internal/gcc/ethr_dw_atomic.h"


#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR 1

static ETHR_INLINE ethr_sint_t *
ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var)
{
    return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var);
}

#if (ETHR_HAVE___atomic_store_n & (2*ETHR_SIZEOF_PTR))

#if (ETHR_GCC_RELAXED_VERSIONS__ & (2*ETHR_SIZEOF_PTR))

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET 1

static ETHR_INLINE void
ethr_native_su_dw_atomic_set(ethr_native_dw_atomic_t *var,
			     ETHR_NATIVE_SU_DW_SINT_T value)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_DW_DBG_ALIGNED__(p);
    __atomic_store_n(p, value, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_VERSIONS__ */
# 122 "../include/internal/gcc/ethr_dw_atomic.h"

#if (ETHR_GCC_RELB_VERSIONS__ & (2*ETHR_SIZEOF_PTR))

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB 1

static ETHR_INLINE void
ethr_native_su_dw_atomic_set_relb(ethr_native_dw_atomic_t *var,
				  ETHR_NATIVE_SU_DW_SINT_T value)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_DW_DBG_ALIGNED__(p);
    __atomic_store_n(p, value, __ATOMIC_RELEASE);
}

#endif /* ETHR_GCC_RELB_VERSIONS__ */
# 137 "../include/internal/gcc/ethr_dw_atomic.h"

#endif /* ETHR_HAVE___atomic_store_n */
# 139 "../include/internal/gcc/ethr_dw_atomic.h"

#if (ETHR_HAVE___atomic_load_n & (2*ETHR_SIZEOF_PTR))

#if (ETHR_GCC_RELAXED_VERSIONS__ & (2*ETHR_SIZEOF_PTR))

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ 1

static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
ethr_native_su_dw_atomic_read(ethr_native_dw_atomic_t *var)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_DW_DBG_ALIGNED__(p);
    return __atomic_load_n(p, __ATOMIC_RELAXED);
}

#endif /* ETHR_GCC_RELAXED_VERSIONS__ */
# 155 "../include/internal/gcc/ethr_dw_atomic.h"

#if ((ETHR_GCC_ACQB_VERSIONS__ & (2*ETHR_SIZEOF_PTR))	\
     & ~ETHR___atomic_load_ACQUIRE_barrier_bug)

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB 1

static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
ethr_native_su_dw_atomic_read_acqb(ethr_native_dw_atomic_t *var)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_DW_DBG_ALIGNED__(p);
    return __atomic_load_n(p, __ATOMIC_ACQUIRE);
}

#endif /* ETHR_GCC_ACQB_VERSIONS__ */
# 170 "../include/internal/gcc/ethr_dw_atomic.h"

#endif /* ETHR_HAVE___atomic_load_n */
# 172 "../include/internal/gcc/ethr_dw_atomic.h"

#if (ETHR_HAVE___atomic_compare_exchange_n & (2*ETHR_SIZEOF_PTR))

#if (ETHR_GCC_RELAXED_MOD_VERSIONS__ & (2*ETHR_SIZEOF_PTR))

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG 1

static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
ethr_native_su_dw_atomic_cmpxchg(ethr_native_dw_atomic_t *var,
				 ETHR_NATIVE_SU_DW_SINT_T new,
				 ETHR_NATIVE_SU_DW_SINT_T exp)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_NATIVE_SU_DW_SINT_T xchg = exp;
    ETHR_DW_DBG_ALIGNED__(p);
    if (__atomic_compare_exchange_n(p,
				    &xchg,
				    new,
				    0,
				    __ATOMIC_RELAXED,
				    __ATOMIC_RELAXED))
	return exp;
    return xchg;
}

#endif /* ETHR_GCC_RELAXED_MOD_VERSIONS__ */
# 198 "../include/internal/gcc/ethr_dw_atomic.h"

#if (ETHR_GCC_ACQB_MOD_VERSIONS__ & (2*ETHR_SIZEOF_PTR))

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB 1

static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
ethr_native_su_dw_atomic_cmpxchg_acqb(ethr_native_dw_atomic_t *var,
				      ETHR_NATIVE_SU_DW_SINT_T new,
				      ETHR_NATIVE_SU_DW_SINT_T exp)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_NATIVE_SU_DW_SINT_T xchg = exp;
    ETHR_DW_DBG_ALIGNED__(p);
    if (__atomic_compare_exchange_n(p,
				    &xchg,
				    new,
				    0,
				    __ATOMIC_ACQUIRE,
				    __ATOMIC_ACQUIRE))
	return exp;
    return xchg;
}

#endif /* ETHR_GCC_ACQB_MOD_VERSIONS__ */
# 222 "../include/internal/gcc/ethr_dw_atomic.h"

#endif /* ETHR_HAVE___atomic_compare_exchange_n */
# 224 "../include/internal/gcc/ethr_dw_atomic.h"

#if ((ETHR_HAVE___sync_val_compare_and_swap & (2*ETHR_SIZEOF_PTR)) \
     & ETHR_GCC_MB_MOD_VERSIONS__)

#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB 1

static ETHR_INLINE ETHR_NATIVE_SU_DW_SINT_T
ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var,
				    ETHR_NATIVE_SU_DW_SINT_T new,
				    ETHR_NATIVE_SU_DW_SINT_T old)
{
    ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var);
    ETHR_DW_DBG_ALIGNED__(p);
    return __sync_val_compare_and_swap(p, old, new);
}

#endif /* ETHR_HAVE___sync_val_compare_and_swap */
# 241 "../include/internal/gcc/ethr_dw_atomic.h"

#endif /* ETHR_TRY_INLINE_FUNCS */
# 243 "../include/internal/gcc/ethr_dw_atomic.h"

#endif /* ETHR_INCLUDE_DW_ATOMIC_IMPL__ */
# 245 "../include/internal/gcc/ethr_dw_atomic.h"
# 350 "../include/internal/gcc/ethread.h" 2
#endif
# 351 "../include/internal/gcc/ethread.h"

#undef ETHR___atomic_load_ACQUIRE_barrier_bug
#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__
#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#undef ETHR_GCC_RELAXED_VERSIONS__
#undef ETHR_GCC_RELB_VERSIONS__
#undef ETHR_GCC_RELB_VERSIONS__
#undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#undef ETHR_GCC_ACQB_MOD_VERSIONS__
#undef ETHR_GCC_RELB_MOD_VERSIONS__
#undef ETHR_GCC_MB_MOD_VERSIONS__

#endif /* ETHREAD_GCC_NATIVE_H__ */
# 366 "../include/internal/gcc/ethread.h"
# 369 "../include/internal/ethread.h" 2
#      endif
# 370 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#      include "libatomic_ops/ethread.h"
#endif /* expanded by -frewrite-includes */
# 370 "../include/internal/ethread.h"
# 1 "../include/internal/libatomic_ops/ethread.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Native atomics ethread support using libatomic_ops
 * Author: Rickard Green
 */

#ifndef ETHREAD_LIBATOMIC_OPS_H__
#define ETHREAD_LIBATOMIC_OPS_H__

#if (defined(ETHR_HAVE_LIBATOMIC_OPS) \
     && ((ETHR_SIZEOF_AO_T == 4 && !defined(ETHR_HAVE_NATIVE_ATOMIC32)) \
	 || (ETHR_SIZEOF_AO_T == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64))))

#if defined(__x86_64__)
#define AO_USE_PENTIUM4_INSTRS
#endif
# 36 "../include/internal/libatomic_ops/ethread.h"

#define ETHR_NATIVE_IMPL__ "libatomic_ops"

#if 0 /* expanded by -frewrite-includes */
#include "atomic_ops.h"
#endif /* expanded by -frewrite-includes */
# 39 "../include/internal/libatomic_ops/ethread.h"
# 40 "../include/internal/libatomic_ops/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include "ethr_membar.h"
#endif /* expanded by -frewrite-includes */
# 40 "../include/internal/libatomic_ops/ethread.h"
# 41 "../include/internal/libatomic_ops/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include "ethr_atomic.h"
#endif /* expanded by -frewrite-includes */
# 41 "../include/internal/libatomic_ops/ethread.h"
# 42 "../include/internal/libatomic_ops/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#include "ethr_dw_atomic.h"
#endif /* expanded by -frewrite-includes */
# 42 "../include/internal/libatomic_ops/ethread.h"
# 43 "../include/internal/libatomic_ops/ethread.h"

#endif
# 45 "../include/internal/libatomic_ops/ethread.h"

#endif
# 47 "../include/internal/libatomic_ops/ethread.h"
# 371 "../include/internal/ethread.h" 2
#if 0 /* expanded by -frewrite-includes */
#      include "gcc/ethread.h"
#endif /* expanded by -frewrite-includes */
# 371 "../include/internal/ethread.h"
# 1 "../include/internal/gcc/ethread.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010-2015. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Native atomic ethread support when using gcc's __atomic
 *              and __sync builtins
 * Author: Rickard Green
 */

#if !defined(ETHREAD_GCC_NATIVE_H__) && ETHR_GCC_COMPILER
#define ETHREAD_GCC_NATIVE_H__

#ifndef ETHR_MEMBAR
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_membar.h"
#endif /* expanded by -frewrite-includes */
# 31 "../include/internal/gcc/ethread.h"
# 32 "../include/internal/gcc/ethread.h"
#endif
# 33 "../include/internal/gcc/ethread.h"

#define ETHR_GCC_VERSIONS_MASK__ 28

#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__
#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#undef ETHR_GCC_RELAXED_VERSIONS__
#undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#undef ETHR_GCC_ACQB_VERSIONS__
#undef ETHR_GCC_ACQB_MOD_VERSIONS__
#undef ETHR_GCC_RELB_VERSIONS__
#undef ETHR_GCC_RELB_MOD_VERSIONS__
#undef ETHR_GCC_MB_MOD_VERSIONS__

/*
 * True GNU GCCs before version 4.8 do not emit a memory barrier
 * after the load in the __atomic_load_n(_, __ATOMIC_ACQUIRE)
 * case (which is needed on most architectures).
 */
#undef ETHR___atomic_load_ACQUIRE_barrier_bug
#if ETHR_GCC_COMPILER != ETHR_GCC_COMPILER_TRUE
/*
 * A gcc compatible compiler. We have no information
 * about the existence of this bug, but we assume
 * that it is not impossible that it could have
 * been "inherited". Therefore, until we are certain
 * that the bug does not exist, we assume that it
 * does.
 */
#  define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__
#elif !ETHR_AT_LEAST_GCC_VSN__(4, 8, 0)
# 65 "../include/internal/gcc/ethread.h"
/* True gcc of version < 4.8, i.e., bug exist... */
#  define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__
#else /* True gcc of version >= 4.8 */
# 68 "../include/internal/gcc/ethread.h"
/*
 * Sizes less than or equal to word size have been fixed,
 * but double word size has not been fixed.
 */
#  if ETHR_SIZEOF_PTR == 8
#    define ETHR___atomic_load_ACQUIRE_barrier_bug \
    (~(8|4) & ETHR_GCC_VERSIONS_MASK__)
#  elif ETHR_SIZEOF_PTR == 4
# 76 "../include/internal/gcc/ethread.h"
#    define ETHR___atomic_load_ACQUIRE_barrier_bug \
    (~4 & ETHR_GCC_VERSIONS_MASK__)
#  else
# 79 "../include/internal/gcc/ethread.h"
#    error word size not supported
#  endif
# 81 "../include/internal/gcc/ethread.h"
#endif
# 82 "../include/internal/gcc/ethread.h"

#define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ 0
#define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 0
#define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 0
#define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 0
#define ETHR_GCC_RELAXED_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#define ETHR_GCC_RELAXED_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__

#if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS
#  define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#else
# 96 "../include/internal/gcc/ethread.h"
/*
 * This is currently the default (on most platforms) since
 * we've seen too many memory barrier bugs produced by gcc...
 */
#  define ETHR_GCC_ACQB_VERSIONS__ 0
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ 0
#  define ETHR_GCC_RELB_VERSIONS__ 0
#  define ETHR_GCC_RELB_MOD_VERSIONS__ 0
#endif
# 105 "../include/internal/gcc/ethread.h"
/*
 * In the general case we do not want any full barrier versions
 * if we can implement more relaxed ones (using __atomic_* builtins).
 * This since the implementations normally need extra memory barrier
 * instructions to implement these. The x86/x86_64 implementations
 * are an exception see below.
 */
#define ETHR_GCC_MB_MOD_VERSIONS__ \
    (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___atomic_compare_exchange_n)

#if ETHR_SIZEOF_PTR == 8
#  define ETHR_GCC_VOLATILE_BIT_MASK__ 12
#elif ETHR_SIZEOF_PTR == 4
# 118 "../include/internal/gcc/ethread.h"
#  define ETHR_GCC_VOLATILE_BIT_MASK__ 4
#endif
# 120 "../include/internal/gcc/ethread.h"

#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__)	\
    || defined(__powerpc__) || defined(__ppc__) || defined(__mips__)	\
    || defined(__alpha__) || defined(__ia64__)

/*
 * Aligned volatile stores and loads of data smaller
 * than or equal to word size are atomic...
 */
#  undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__
#  define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ ETHR_GCC_VOLATILE_BIT_MASK__
#  undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#  define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ ETHR_GCC_VOLATILE_BIT_MASK__

#elif defined(__arm__)
# 135 "../include/internal/gcc/ethread.h"

/* volatile stores are problematic on some machines... */
#  undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#  define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ ETHR_GCC_VOLATILE_BIT_MASK__

#endif
# 141 "../include/internal/gcc/ethread.h"

#if defined(__ia64__)

/* Volatile stores produce stores with release barriers. */
#  undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#  define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ ETHR_GCC_VOLATILE_BIT_MASK__

/* Volatile loads produce loads with acquire barrier. */
#  undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#  define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ ETHR_GCC_VOLATILE_BIT_MASK__

/*
 * We trust gcc to produce acquire/release barriers on itanium.
 * Since all atomic ops also have at least acquire or release
 * barriers (also when passed the relaxed memory model) it
 * would be very inefficient not to use these as native
 * barriers on Itanium.
 */
#  undef ETHR_GCC_ACQB_VERSIONS__
#  define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  undef ETHR_GCC_ACQB_MOD_VERSIONS__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  undef ETHR_GCC_RELB_VERSIONS__
#  define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__
#  undef ETHR_GCC_RELB_MOD_VERSIONS__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__

/*
 * Itanium is not effected by the load acquire
 * bug since the barrier is part of the instruction
 * on Itanium (ld.acq), and not a separate instruction
 * as on most platforms.
 */
#  undef ETHR___atomic_load_ACQUIRE_barrier_bug
#  define ETHR___atomic_load_ACQUIRE_barrier_bug 0

/*
 * No point exposing relaxed versions since they are
 * implemended using either acquire or release
 * barriers.
 */
#  undef ETHR_GCC_RELAXED_VERSIONS__
#  define ETHR_GCC_RELAXED_VERSIONS__ 0

/* #endif defined(__ia64__) */
#elif defined(__i386__) || defined(__x86_64__)
# 187 "../include/internal/gcc/ethread.h"

/*
 * Want full barrier versions of all modification
 * operations since all of these are implemented
 * using locked instructions implying full memory
 * barriers.
 */
#  undef ETHR_GCC_MB_MOD_VERSIONS__
#  define ETHR_GCC_MB_MOD_VERSIONS__ ETHR_HAVE___sync_val_compare_and_swap

/*
 * No point exposing acquire/release versions
 * when we got full memory barrier versions
 * of modification operations since all of these
 * are implemented using locked instructions
 * implying full memory barriers.
 */
#  if ETHR_GCC_ACQB_MOD_VERSIONS__
#    undef ETHR_GCC_ACQB_MOD_VERSIONS__
#    define ETHR_GCC_ACQB_MOD_VERSIONS__ \
    (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___sync_val_compare_and_swap)
#  endif
# 209 "../include/internal/gcc/ethread.h"
#  if ETHR_GCC_RELB_MOD_VERSIONS__
#    undef ETHR_GCC_RELB_MOD_VERSIONS__
#    define ETHR_GCC_RELB_MOD_VERSIONS__ \
    (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___sync_val_compare_and_swap)
#  endif
# 214 "../include/internal/gcc/ethread.h"

#  ifdef ETHR_X86_OUT_OF_ORDER

/* See above... */
#    undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#    define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0

#  else /* !ETHR_X86_OUT_OF_ORDER, i.e., we don't use any x86-OOO instructions... */
# 222 "../include/internal/gcc/ethread.h"

/*
 * Not effected by the load acquire barrier bug,
 * since no barrier at all is needed for a load
 * acquire...
 */
#    undef ETHR___atomic_load_ACQUIRE_barrier_bug
#    define ETHR___atomic_load_ACQUIRE_barrier_bug 0

/* Stores imply release barriers semantics. */
#    undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#    define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ ETHR_GCC_VOLATILE_BIT_MASK__

/* Loads imply acquire barrier semantics. */
#    undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#    define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ ETHR_GCC_VOLATILE_BIT_MASK__

/*
 * Trust load acquire and store release for sizes
 * where volatile operation implies these barrier
 * semantics since no barriers are needed.
 */
#    if !ETHR_GCC_ACQB_VERSIONS__
#      undef ETHR_GCC_ACQB_VERSIONS__
#      define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VOLATILE_BIT_MASK__
#    endif
# 248 "../include/internal/gcc/ethread.h"
#    if !ETHR_GCC_RELB_VERSIONS__
#      undef ETHR_GCC_RELB_VERSIONS__
#      define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VOLATILE_BIT_MASK__
#    endif
# 252 "../include/internal/gcc/ethread.h"

/*
 * No point exposing relaxed versions at all since
 * all mod operations are implemented with locked
 * instructions implying full memory barriers and
 * volatile store and load imply release and
 * acquire barrier semantics.
 */
#    undef ETHR_GCC_RELAXED_VERSIONS__
#    define ETHR_GCC_RELAXED_VERSIONS__ 0

#  endif /* !ETHR_X86_OUT_OF_ORDER */
# 264 "../include/internal/gcc/ethread.h"

/* #endif defined(__i386__) || defined(__x86_64__) */
#elif defined(__powerpc__) || defined(__ppc__)
# 267 "../include/internal/gcc/ethread.h"

#  if !defined(ETHR_PPC_HAVE_LWSYNC)
/*
 * Release barriers are typically implemented using
 * the lwsync instruction. We want our runtime
 * configure test to determine if the lwsync
 * instruction is available on the system or not
 * before we use it. Therefore, do not implement any
 * native ops using the __ATOMIC_RELEASE model.
 */
#    undef ETHR_GCC_RELB_VERSIONS__
#    define ETHR_GCC_RELB_VERSIONS__ 0
#    if defined(ETHR_GCC_IMPLEMENT_ACQB_USING_LWSYNC)
/*
 * Acquire barriers are usually implemented by other means
 * than lwsync, but can be implemented using lwsync. Define
 * ETHR_GCC_IMPLEMENT_ACQB_USING_LWSYNC if acquire barriers
 * are implemented using lwsync.
 */
#      undef ETHR_GCC_ACQB_VERSIONS__
#      define ETHR_GCC_ACQB_VERSIONS__ 0
#    endif
# 289 "../include/internal/gcc/ethread.h"
#  endif
# 290 "../include/internal/gcc/ethread.h"

#endif /* defined(__powerpc__) || defined(__ppc__) */
# 292 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELAXED_VERSIONS__
#  undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#  define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0
#endif
# 297 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_ACQB_VERSIONS__
#  undef ETHR_GCC_ACQB_MOD_VERSIONS__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ 0
#endif
# 302 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELB_VERSIONS__
#  undef ETHR_GCC_RELB_MOD_VERSIONS__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ 0
#endif
# 307 "../include/internal/gcc/ethread.h"

#if !defined(ETHR_HAVE_NATIVE_ATOMIC32)
#  define ETHR_ATOMIC_WANT_32BIT_IMPL__
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_atomic.h"
#endif /* expanded by -frewrite-includes */
# 310 "../include/internal/gcc/ethread.h"
# 311 "../include/internal/gcc/ethread.h"
#endif
# 312 "../include/internal/gcc/ethread.h"

#if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64)
#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_atomic.h"
#endif /* expanded by -frewrite-includes */
# 315 "../include/internal/gcc/ethread.h"
# 316 "../include/internal/gcc/ethread.h"
#endif
# 317 "../include/internal/gcc/ethread.h"

#if defined(__x86_64__)
/*
 * No instructions available for native implementation
 * of these for dw-atomics...
 */
#  undef ETHR_GCC_RELAXED_VERSIONS__
#  define ETHR_GCC_RELAXED_VERSIONS__ 0
#  undef ETHR_GCC_ACQB_VERSIONS__
#  define ETHR_GCC_ACQB_VERSIONS__ 0
#  undef ETHR_GCC_RELB_VERSIONS__
#  define ETHR_GCC_RELB_VERSIONS__ 0
#endif
# 330 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELAXED_VERSIONS__
#  undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#  define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0
#endif
# 335 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_ACQB_VERSIONS__
#  undef ETHR_GCC_ACQB_MOD_VERSIONS__
#  define ETHR_GCC_ACQB_MOD_VERSIONS__ 0
#endif
# 340 "../include/internal/gcc/ethread.h"

#if !ETHR_GCC_RELB_VERSIONS__
#  undef ETHR_GCC_RELB_MOD_VERSIONS__
#  define ETHR_GCC_RELB_MOD_VERSIONS__ 0
#endif
# 345 "../include/internal/gcc/ethread.h"

#if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \
     && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \
     && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128)))
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_dw_atomic.h"
#endif /* expanded by -frewrite-includes */
# 349 "../include/internal/gcc/ethread.h"
# 350 "../include/internal/gcc/ethread.h"
#endif
# 351 "../include/internal/gcc/ethread.h"

#undef ETHR___atomic_load_ACQUIRE_barrier_bug
#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__
#undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__
#undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__
#undef ETHR_GCC_RELAXED_VERSIONS__
#undef ETHR_GCC_RELB_VERSIONS__
#undef ETHR_GCC_RELB_VERSIONS__
#undef ETHR_GCC_RELAXED_MOD_VERSIONS__
#undef ETHR_GCC_ACQB_MOD_VERSIONS__
#undef ETHR_GCC_RELB_MOD_VERSIONS__
#undef ETHR_GCC_MB_MOD_VERSIONS__

#endif /* ETHREAD_GCC_NATIVE_H__ */
# 366 "../include/internal/gcc/ethread.h"
# 372 "../include/internal/ethread.h" 2
#    endif
# 373 "../include/internal/ethread.h"
#  elif defined(ETHR_HAVE_LIBATOMIC_OPS)
# 374 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#    include "libatomic_ops/ethread.h"
#endif /* expanded by -frewrite-includes */
# 374 "../include/internal/ethread.h"
# 375 "../include/internal/ethread.h"
#  elif defined(ETHR_WIN32_THREADS)
# 376 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#    include "win/ethread.h"
#endif /* expanded by -frewrite-includes */
# 376 "../include/internal/ethread.h"
# 377 "../include/internal/ethread.h"
#  endif
# 378 "../include/internal/ethread.h"
#endif /* !ETHR_DISABLE_NATIVE_IMPLS */
# 379 "../include/internal/ethread.h"

#if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64) && !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(ETHR_SMP_REQUIRE_NATIVE_IMPLS)
#error "No native ethread implementation found. If you want to use fallbacks you have to disable native ethread support with configure."
#endif
# 383 "../include/internal/ethread.h"

#if 0 /* expanded by -frewrite-includes */
#include "ethr_atomics.h" /* The atomics API */
#endif /* expanded by -frewrite-includes */
# 384 "../include/internal/ethread.h"
# 1 "../include/internal/ethr_atomics.h" 1
/*
 * --------------- DO NOT EDIT THIS FILE! ---------------
 * This file was automatically generated by the
 * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
 * If you need to make changes, edit the script and
 * regenerate this file.
 * --------------- DO NOT EDIT THIS FILE! ---------------
 */

/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: The ethread atomics API
 * Author: Rickard Green
 */

/*
 * This file maps native atomic implementations to ethread
 * API atomics. If no native atomic implementation
 * is available, a less efficient fallback is used instead.
 * The API consists of 32-bit size, word size (pointer size),
 * and double word size atomics.
 *
 * The following atomic operations are implemented for
 * 32-bit size, and word size atomics:
 * - cmpxchg
 * - xchg
 * - set
 * - init
 * - add_read
 * - read
 * - inc_read
 * - dec_read
 * - add
 * - inc
 * - dec
 * - read_band
 * - read_bor
 *
 * The following atomic operations are implemented for
 * double word size atomics:
 * - cmpxchg
 * - set
 * - read
 * - init
 *
 * Appart from a function implementing the atomic operation
 * with unspecified memory barrier semantics, there are
 * functions implementing each operation with the following
 * implied memory barrier semantics:
 * - mb   - Full memory barrier. Orders both loads, and
 *          stores before, and after the atomic operation.
 *          No load or store is allowed to be reordered
 *          over the atomic operation.
 * - relb - Release barrier. Orders both loads, and
 *          stores appearing *before* the atomic
 *          operation. These are not allowed to be
 *          reordered over the atomic operation.
 * - acqb - Acquire barrier. Orders both loads, and stores
 *          appearing *after* the atomic operation. These
 *          are not allowed to be reordered over the
 *          atomic operation.
 * - wb   - Write barrier. Orders *only* stores. These are
 *          not allowed to be reordered over the barrier.
 *          Store in atomic operation is ordered *after*
 *          the barrier.
 * - rb   - Read barrier. Orders *only* loads. These are
 *          not allowed to be reordered over the barrier.
 *          Load in atomic operation is ordered *before*
 *          the barrier. 
 * - ddrb - Data dependency read barrier. Orders *only*
 *          loads according to data dependency across the
 *          barrier. Load in atomic operation is ordered
 *          before the barrier.
 *
 * We implement all of these operation/barrier
 * combinations, regardless of whether they are useful
 * or not (some of them are useless).
 *
 * Double word size atomic functions are on the followning
 * form:
 *   ethr_dw_atomic_<OP>[_<BARRIER>]
 *
 * Word size atomic functions are on the followning
 * form:
 *   ethr_atomic_<OP>[_<BARRIER>]
 *
 * 32-bit size atomic functions are on the followning
 * form:
 *   ethr_atomic32_<OP>[_<BARRIER>]
 *
 * Apart from the operation/barrier functions
 * described above also 'addr' functions are implemented
 * which return the actual memory address used of the
 * atomic variable. The 'addr' functions have no barrier
 * versions.
 *
 * The native atomic implementation does not need to
 * implement all operation/barrier combinations.
 * Functions that have no native implementation will be
 * constructed from existing native functionality. These
 * functions will perform the wanted operation and will
 * produce sufficient memory barriers, but may
 * in some cases be less efficient than pure native
 * versions.
 *
 * When we create ethread API operation/barrier functions by
 * adding barriers before and after native operations it is
 * assumed that:
 * - A native read operation begins, and ends with a load.
 * - A native set operation begins, and ends with a store.
 * - An init operation begins with either a load, or a store,
 *   and ends with either a load, or a store.
 * - All other operations begins with a load, and ends with
 *   either a load, or a store.
 *
 * This is the minimum functionality that a native
 * implementation needs to provide:
 *
 * - Functions that need to be implemented:
 *
 *   - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
 *   - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
 *     (at least one cmpxchg of optional barrier)
 *
 * - Macros that needs to be defined:
 *
 *   A macro informing about the presence of the native
 *   implementation:
 *
 *   - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
 *
 *   A macro naming (a string constant) the implementation:
 *
 *   - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
 *
 *   Each implemented native atomic function has to
 *   be accompanied by a defined macro on the following
 *   form informing about its presence:
 *
 *   - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
 *
 *   A (sparc-v9 style) membar macro:
 *
 *   - ETHR_MEMBAR(B)
 *
 *     Which takes a combination of the following macros
 *     or:ed (using |) together:
 *
 *     - ETHR_LoadLoad
 *     - ETHR_LoadStore
 *     - ETHR_StoreLoad
 *     - ETHR_StoreStore
 *
 */

#ifndef ETHR_ATOMICS_H__
#define ETHR_ATOMICS_H__

#undef ETHR_AMC_FALLBACK__
#undef ETHR_AMC_NO_ATMCS__
#undef ETHR_AMC_ATMC_T__
#undef ETHR_AMC_ATMC_FUNC__

/* -- 32-bit atomics -- */

#undef ETHR_NAINT32_T__
#undef ETHR_NATMC32_FUNC__
#undef ETHR_NATMC32_ADDR_FUNC__
#undef ETHR_NATMC32_BITS__
#if defined(ETHR_HAVE_NATIVE_ATOMIC32)
#    define ETHR_NEED_NATMC32_ADDR
#  define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr
typedef ethr_native_atomic32_t ethr_atomic32_t;
#  define ETHR_NAINT32_T__ ethr_sint32_t
#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X
#  define ETHR_NATMC32_BITS__ 32
#elif defined(ETHR_HAVE_NATIVE_ATOMIC64)
# 197 "../include/internal/ethr_atomics.h"
#  define ETHR_NEED_NATMC64_ADDR
#ifdef ETHR_BIGENDIAN
#  define ETHR_NATMC32_ADDR_FUNC__(VAR) \
  (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
#else
# 202 "../include/internal/ethr_atomics.h"
#  define ETHR_NATMC32_ADDR_FUNC__(VAR) \
  ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
#endif
# 205 "../include/internal/ethr_atomics.h"
typedef ethr_native_atomic64_t ethr_atomic32_t;
#  define ETHR_NAINT32_T__ ethr_sint64_t
#  define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X
#  define ETHR_NATMC32_BITS__ 64
#else
# 210 "../include/internal/ethr_atomics.h"
/*
 * No native atomics usable for 32-bits atomics :(
 * Use fallback...
 */
typedef ethr_sint32_t ethr_atomic32_t;
#endif
# 216 "../include/internal/ethr_atomics.h"

#undef ETHR_ATMC32_INLINE__
#ifdef ETHR_NATMC32_BITS__
#  ifdef ETHR_TRY_INLINE_FUNCS
#    define ETHR_ATMC32_INLINE__
#  endif
# 222 "../include/internal/ethr_atomics.h"
#  define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS
#endif
# 224 "../include/internal/ethr_atomics.h"

#if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
#  define ETHR_NEED_ATMC32_PROTOTYPES__
#endif
# 228 "../include/internal/ethr_atomics.h"

#ifndef ETHR_INLINE_ATMC32_FUNC_NAME_
#  define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X
#endif
# 232 "../include/internal/ethr_atomics.h"

#undef ETHR_ATMC32_FUNC__
#define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)


/* -- Word size atomics -- */

#undef ETHR_NEED_NATMC32_ADDR
#undef ETHR_NEED_NATMC64_ADDR

#undef ETHR_NAINT_T__
#undef ETHR_NATMC_FUNC__
#undef ETHR_NATMC_ADDR_FUNC__
#undef ETHR_NATMC_BITS__
#if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
#  ifndef ETHR_NEED_NATMC64_ADDR
#    define ETHR_NEED_NATMC64_ADDR
#  endif
# 250 "../include/internal/ethr_atomics.h"
#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr
typedef ethr_native_atomic64_t ethr_atomic_t;
#  define ETHR_NAINT_T__ ethr_sint64_t
#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#  define ETHR_NATMC_BITS__ 64
#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32)
# 256 "../include/internal/ethr_atomics.h"
#  ifndef ETHR_NEED_NATMC64_ADDR
#    define ETHR_NEED_NATMC32_ADDR
#  endif
# 259 "../include/internal/ethr_atomics.h"
#  define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr
typedef ethr_native_atomic32_t ethr_atomic_t;
#  define ETHR_NAINT_T__ ethr_sint32_t
#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
#  define ETHR_NATMC_BITS__ 32
#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
# 265 "../include/internal/ethr_atomics.h"
#  ifndef ETHR_NEED_NATMC64_ADDR
#    define ETHR_NEED_NATMC64_ADDR
#  endif
# 268 "../include/internal/ethr_atomics.h"
#ifdef ETHR_BIGENDIAN
#  define ETHR_NATMC_ADDR_FUNC__(VAR) \
  (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1)
#else
# 272 "../include/internal/ethr_atomics.h"
#  define ETHR_NATMC_ADDR_FUNC__(VAR) \
  ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR)))
#endif
# 275 "../include/internal/ethr_atomics.h"
typedef ethr_native_atomic64_t ethr_atomic_t;
#  define ETHR_NATMC_T__ ethr_native_atomic64_t
#  define ETHR_NAINT_T__ ethr_sint64_t
#  define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#  define ETHR_NATMC_BITS__ 64
#else
# 281 "../include/internal/ethr_atomics.h"
/*
 * No native atomics usable for pointer size atomics :(
 * Use fallback...
 */

#  if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
#    define ETHR_AMC_FALLBACK__
#    define ETHR_AMC_NO_ATMCS__ 2
#    define ETHR_AMC_SINT_T__ ethr_sint32_t
#    define ETHR_AMC_ATMC_T__ ethr_atomic32_t
#    define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
typedef struct {
    ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
} ethr_amc_t;
typedef struct {
    ethr_amc_t amc;
    ethr_sint_t sint;
} ethr_atomic_t;
#  else /* locked fallback */
# 300 "../include/internal/ethr_atomics.h"
typedef ethr_sint_t ethr_atomic_t;
#  endif
# 302 "../include/internal/ethr_atomics.h"
#endif
# 303 "../include/internal/ethr_atomics.h"

#undef ETHR_ATMC_INLINE__
#ifdef ETHR_NATMC_BITS__
#  ifdef ETHR_TRY_INLINE_FUNCS
#    define ETHR_ATMC_INLINE__
#  endif
# 309 "../include/internal/ethr_atomics.h"
#  define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS
#endif
# 311 "../include/internal/ethr_atomics.h"

#if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
#  define ETHR_NEED_ATMC_PROTOTYPES__
#endif
# 315 "../include/internal/ethr_atomics.h"

#ifndef ETHR_INLINE_ATMC_FUNC_NAME_
#  define ETHR_INLINE_ATMC_FUNC_NAME_(X) X
#endif
# 319 "../include/internal/ethr_atomics.h"

#undef ETHR_ATMC_FUNC__
#define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)

/* -- Double word atomics -- */

#undef ETHR_SU_DW_NAINT_T__
#undef ETHR_SU_DW_NATMC_FUNC__
#undef ETHR_SU_DW_NATMC_ADDR_FUNC__
#undef ETHR_DW_NATMC_FUNC__
#undef ETHR_DW_NATMC_ADDR_FUNC__
#undef ETHR_DW_NATMC_BITS__
#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
#  define ETHR_NEED_DW_NATMC_ADDR
#  define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr
#  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t
#  define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X
#  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X
#  if ETHR_SIZEOF_PTR == 8
#    define ETHR_DW_NATMC_BITS__ 128
#  elif ETHR_SIZEOF_PTR == 4
# 340 "../include/internal/ethr_atomics.h"
#    define ETHR_DW_NATMC_BITS__ 64
#  else
# 342 "../include/internal/ethr_atomics.h"
#    error "Word size not supported"
#  endif
# 344 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_NATIVE_SU_DW_SINT_T
#    define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T
#  endif
# 347 "../include/internal/ethr_atomics.h"
#elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)
# 348 "../include/internal/ethr_atomics.h"
#  define ETHR_HAVE_NATIVE_SU_DW_ATOMIC
#  ifndef ETHR_NEED_NATMC64_ADDR
#    define ETHR_NEED_NATMC64_ADDR
#  endif
# 352 "../include/internal/ethr_atomics.h"
#  define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \
  ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR)))
#  define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t
#  define ETHR_SU_DW_NAINT_T__ ethr_sint64_t
#  define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
#  define ETHR_DW_NATMC_BITS__ 64
#endif
# 359 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X ## _fallback__
#else
# 363 "../include/internal/ethr_atomics.h"
#define ETHR_DW_ATOMIC_FUNC__(X) ethr_dw_atomic_ ## X
#endif
# 365 "../include/internal/ethr_atomics.h"

#if !defined(ETHR_DW_NATMC_BITS__) || defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
#  define ETHR_NEED_DW_FALLBACK__
#endif
# 369 "../include/internal/ethr_atomics.h"

#if defined(ETHR_NEED_DW_FALLBACK__)
/*
 * No native atomics usable for double word atomics :(
 * Use fallback...
 */

#  ifndef ETHR_AMC_FALLBACK__
#    if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
#      define ETHR_AMC_FALLBACK__
#      define ETHR_AMC_NO_ATMCS__ 1
#      define ETHR_AMC_SINT_T__ ethr_sint_t
#      define ETHR_AMC_ATMC_T__ ethr_atomic_t
#      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X)
#    elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
# 384 "../include/internal/ethr_atomics.h"
#      define ETHR_AMC_FALLBACK__
#      define ETHR_AMC_NO_ATMCS__ 2
#      define ETHR_AMC_SINT_T__ ethr_sint32_t
#      define ETHR_AMC_ATMC_T__ ethr_atomic32_t
#      define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)
#    endif
# 390 "../include/internal/ethr_atomics.h"
#    ifdef ETHR_AMC_FALLBACK__
typedef struct {
    ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__];
} ethr_amc_t;
#    endif
# 395 "../include/internal/ethr_atomics.h"
#  endif
# 396 "../include/internal/ethr_atomics.h"

typedef struct {
#ifdef ETHR_AMC_FALLBACK__
    ethr_amc_t amc;
#endif
# 401 "../include/internal/ethr_atomics.h"
    ethr_sint_t sint[2];
} ethr_dw_atomic_fallback_t;

#endif
# 405 "../include/internal/ethr_atomics.h"

typedef union {
#ifdef ETHR_NATIVE_DW_ATOMIC_T__
    ETHR_NATIVE_DW_ATOMIC_T__ native;
#endif
# 410 "../include/internal/ethr_atomics.h"
#ifdef ETHR_NEED_DW_FALLBACK__
    ethr_dw_atomic_fallback_t fallback;
#endif
# 413 "../include/internal/ethr_atomics.h"
    ethr_sint_t sint[2];
} ethr_dw_atomic_t;

typedef union {
#ifdef ETHR_SU_DW_NAINT_T__
    ETHR_SU_DW_NAINT_T__ dw_sint;
#endif
# 420 "../include/internal/ethr_atomics.h"
    ethr_sint_t sint[2];
} ethr_dw_sint_t;

#ifdef ETHR_BIGENDIAN
#  define ETHR_DW_SINT_LOW_WORD 1
#  define ETHR_DW_SINT_HIGH_WORD 0
#else
# 427 "../include/internal/ethr_atomics.h"
#  define ETHR_DW_SINT_LOW_WORD 0
#  define ETHR_DW_SINT_HIGH_WORD 1
#endif
# 430 "../include/internal/ethr_atomics.h"

#undef ETHR_DW_ATMC_INLINE__
#ifdef ETHR_DW_NATMC_BITS__
#  ifdef ETHR_TRY_INLINE_FUNCS
#    define ETHR_ATMC32_INLINE__
#  endif
# 436 "../include/internal/ethr_atomics.h"
#  define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS
#endif
# 438 "../include/internal/ethr_atomics.h"

#if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
#  define ETHR_NEED_DW_ATMC_PROTOTYPES__
#endif
# 442 "../include/internal/ethr_atomics.h"

#ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_
#  define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X
#endif
# 446 "../include/internal/ethr_atomics.h"

#undef ETHR_DW_ATMC_FUNC__
#define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X)

#if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__)
int ethr_have_native_dw_atomic(void);
#endif
# 453 "../include/internal/ethr_atomics.h"
#if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)
static ETHR_INLINE int
ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void)
{
#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    return ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__;
#elif defined(ETHR_DW_NATMC_BITS__)
# 460 "../include/internal/ethr_atomics.h"
    return 1;
#else
# 462 "../include/internal/ethr_atomics.h"
    return 0;
#endif
# 464 "../include/internal/ethr_atomics.h"
}
#endif
# 466 "../include/internal/ethr_atomics.h"

/* -- Misc -- */

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
/*
 * Unusual values are used by read() fallbacks implemented via cmpxchg().
 * We want to use an unusual value in hope that it is more efficient
 * not to match the value in memory.
 *
 * - Negative integer values are probably more unusual.
 * - Very large absolute integer values are probably more unusual.
 * - Odd pointers are probably more unusual (only char pointers can be odd).
 */
#  define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181)
#  if ETHR_SIZEOF_PTR == 4
#    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__)
#  elif ETHR_SIZEOF_PTR == 8
# 483 "../include/internal/ethr_atomics.h"
#    define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L)
#  else
# 485 "../include/internal/ethr_atomics.h"
#    error "Word size not supported"
#  endif
# 487 "../include/internal/ethr_atomics.h"
#  if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR)
#    error "No ethr_native_dw_atomic_addr() available"
#  endif
# 490 "../include/internal/ethr_atomics.h"
#  if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR)
#    error "No ethr_native_atomic32_addr() available"
#  endif
# 493 "../include/internal/ethr_atomics.h"
#  if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR)
#    error "No ethr_native_atomic64_addr() available"
#  endif
# 496 "../include/internal/ethr_atomics.h"
#endif
# 497 "../include/internal/ethr_atomics.h"

#if defined(__GNUC__)
#  ifndef ETHR_COMPILER_BARRIER
#    define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory")
#  endif
# 502 "../include/internal/ethr_atomics.h"
#elif defined(ETHR_WIN32_THREADS)
# 503 "../include/internal/ethr_atomics.h"
#  ifndef ETHR_COMPILER_BARRIER
#if 0 /* expanded by -frewrite-includes */
#    include <intrin.h>
#endif /* expanded by -frewrite-includes */
# 504 "../include/internal/ethr_atomics.h"
# 505 "../include/internal/ethr_atomics.h"
#    pragma intrinsic(_ReadWriteBarrier)
#    define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
#  endif
# 508 "../include/internal/ethr_atomics.h"
#endif
# 509 "../include/internal/ethr_atomics.h"

void ethr_compiler_barrier_fallback(void);
#ifndef ETHR_COMPILER_BARRIER
#  define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback()
#endif
# 514 "../include/internal/ethr_atomics.h"

int ethr_init_atomics(void);

/* info */
char **ethr_native_atomic32_ops(void);
char **ethr_native_atomic64_ops(void);
char **ethr_native_dw_atomic_ops(void);
char **ethr_native_su_dw_atomic_ops(void);

#if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__)
/*
 * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only,
 * i.e. when no native atomic implementation exist and only our lock
 * based atomic fallback is used, a noop is sufficient.
 */
#  undef ETHR_MEMORY_BARRIER
#  undef ETHR_WRITE_MEMORY_BARRIER
#  undef ETHR_READ_MEMORY_BARRIER
#  undef ETHR_READ_DEPEND_MEMORY_BARRIER
#  undef ETHR_MEMBAR
#  define ETHR_MEMBAR(B) do { } while (0)
#endif
# 536 "../include/internal/ethr_atomics.h"

#ifndef ETHR_MEMBAR
#  error "No ETHR_MEMBAR defined"
#endif
# 540 "../include/internal/ethr_atomics.h"

#define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore)
#define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore)
#define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad)
#ifdef ETHR_READ_DEPEND_MEMORY_BARRIER
#  undef ETHR_ORDERED_READ_DEPEND
#else
# 547 "../include/internal/ethr_atomics.h"
#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
#  define ETHR_ORDERED_READ_DEPEND
#endif
# 550 "../include/internal/ethr_atomics.h"


/* ---------- Double word size atomic implementation ---------- */


#ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__
ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var);
int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
#endif
# 616 "../include/internal/ethr_atomics.h"
#endif /* ETHR_NEED_DW_ATMC_PROTOTYPES__ */
# 617 "../include/internal/ethr_atomics.h"

#if (defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) \
     && (defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))

#if !defined(ETHR_DW_NATMC_BITS__)
#  error "Missing native atomic implementation"
#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC)  || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
# 624 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
#    define ETHR_HAVE_DW_NATMC_CMPXCHG 1
#  endif
# 629 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
#  endif
# 632 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
#    define ETHR_HAVE_DW_NATMC_CMPXCHG_RB 1
#  endif
# 637 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
#  endif
# 640 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
#    define ETHR_HAVE_DW_NATMC_CMPXCHG_WB 1
#  endif
# 645 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
#  endif
# 648 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
#    define ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB 1
#  endif
# 653 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
#  endif
# 656 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
#    define ETHR_HAVE_DW_NATMC_CMPXCHG_RELB 1
#  endif
# 661 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
#  endif
# 664 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
#    define ETHR_HAVE_DW_NATMC_CMPXCHG_MB 1
#  endif
# 669 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
#  endif
# 672 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_SET
#  undef ETHR_HAVE_DW_NATMC_SET
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
#    define ETHR_HAVE_DW_NATMC_SET 1
#  endif
# 677 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
#    define ETHR_HAVE_SU_DW_NATMC_SET 1
#  endif
# 680 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_SET_RB
#  undef ETHR_HAVE_DW_NATMC_SET_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
#    define ETHR_HAVE_DW_NATMC_SET_RB 1
#  endif
# 685 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
#    define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
#  endif
# 688 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_SET_WB
#  undef ETHR_HAVE_DW_NATMC_SET_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
#    define ETHR_HAVE_DW_NATMC_SET_WB 1
#  endif
# 693 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
#    define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
#  endif
# 696 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
#  undef ETHR_HAVE_DW_NATMC_SET_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
#    define ETHR_HAVE_DW_NATMC_SET_ACQB 1
#  endif
# 701 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
#  endif
# 704 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
#  undef ETHR_HAVE_DW_NATMC_SET_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
#    define ETHR_HAVE_DW_NATMC_SET_RELB 1
#  endif
# 709 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
#    define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
#  endif
# 712 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_SET_MB
#  undef ETHR_HAVE_DW_NATMC_SET_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
#    define ETHR_HAVE_DW_NATMC_SET_MB 1
#  endif
# 717 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
#    define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
#  endif
# 720 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_READ
#  undef ETHR_HAVE_DW_NATMC_READ
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
#    define ETHR_HAVE_DW_NATMC_READ 1
#  endif
# 725 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
#    define ETHR_HAVE_SU_DW_NATMC_READ 1
#  endif
# 728 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_READ_RB
#  undef ETHR_HAVE_DW_NATMC_READ_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
#    define ETHR_HAVE_DW_NATMC_READ_RB 1
#  endif
# 733 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
#    define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
#  endif
# 736 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_READ_WB
#  undef ETHR_HAVE_DW_NATMC_READ_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
#    define ETHR_HAVE_DW_NATMC_READ_WB 1
#  endif
# 741 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
#    define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
#  endif
# 744 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
#  undef ETHR_HAVE_DW_NATMC_READ_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
#    define ETHR_HAVE_DW_NATMC_READ_ACQB 1
#  endif
# 749 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
#  endif
# 752 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
#  undef ETHR_HAVE_DW_NATMC_READ_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
#    define ETHR_HAVE_DW_NATMC_READ_RELB 1
#  endif
# 757 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
#    define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
#  endif
# 760 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_READ_MB
#  undef ETHR_HAVE_DW_NATMC_READ_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
#    define ETHR_HAVE_DW_NATMC_READ_MB 1
#  endif
# 765 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
#    define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
#  endif
# 768 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_INIT
#  undef ETHR_HAVE_DW_NATMC_INIT
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
#    define ETHR_HAVE_DW_NATMC_INIT 1
#  endif
# 773 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
#    define ETHR_HAVE_SU_DW_NATMC_INIT 1
#  endif
# 776 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
#  undef ETHR_HAVE_DW_NATMC_INIT_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
#    define ETHR_HAVE_DW_NATMC_INIT_RB 1
#  endif
# 781 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
#  endif
# 784 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
#  undef ETHR_HAVE_DW_NATMC_INIT_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
#    define ETHR_HAVE_DW_NATMC_INIT_WB 1
#  endif
# 789 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
#  endif
# 792 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
#  undef ETHR_HAVE_DW_NATMC_INIT_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
#    define ETHR_HAVE_DW_NATMC_INIT_ACQB 1
#  endif
# 797 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
#  endif
# 800 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
#  undef ETHR_HAVE_DW_NATMC_INIT_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
#    define ETHR_HAVE_DW_NATMC_INIT_RELB 1
#  endif
# 805 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
#  endif
# 808 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
#  undef ETHR_HAVE_DW_NATMC_INIT_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
#    define ETHR_HAVE_DW_NATMC_INIT_MB 1
#  endif
# 813 "../include/internal/ethr_atomics.h"
#  ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
#  endif
# 816 "../include/internal/ethr_atomics.h"
#elif ETHR_DW_NATMC_BITS__ == 64
# 817 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG 1
#  endif
# 822 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RB
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB 1
#  endif
# 827 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_WB
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB 1
#  endif
# 832 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB 1
#  endif
# 837 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_RELB
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB 1
#  endif
# 842 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_CMPXCHG_MB
#  undef ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
#    define ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB 1
#  endif
# 847 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_SET
#  undef ETHR_HAVE_SU_DW_NATMC_SET
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
#    define ETHR_HAVE_SU_DW_NATMC_SET 1
#  endif
# 852 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_SET_RB
#  undef ETHR_HAVE_SU_DW_NATMC_SET_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
#    define ETHR_HAVE_SU_DW_NATMC_SET_RB 1
#  endif
# 857 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_SET_WB
#  undef ETHR_HAVE_SU_DW_NATMC_SET_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
#    define ETHR_HAVE_SU_DW_NATMC_SET_WB 1
#  endif
# 862 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_SET_ACQB
#  undef ETHR_HAVE_SU_DW_NATMC_SET_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_SET_ACQB 1
#  endif
# 867 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_SET_RELB
#  undef ETHR_HAVE_SU_DW_NATMC_SET_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
#    define ETHR_HAVE_SU_DW_NATMC_SET_RELB 1
#  endif
# 872 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_SET_MB
#  undef ETHR_HAVE_SU_DW_NATMC_SET_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
#    define ETHR_HAVE_SU_DW_NATMC_SET_MB 1
#  endif
# 877 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_READ
#  undef ETHR_HAVE_SU_DW_NATMC_READ
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
#    define ETHR_HAVE_SU_DW_NATMC_READ 1
#  endif
# 882 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_READ_RB
#  undef ETHR_HAVE_SU_DW_NATMC_READ_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
#    define ETHR_HAVE_SU_DW_NATMC_READ_RB 1
#  endif
# 887 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_READ_WB
#  undef ETHR_HAVE_SU_DW_NATMC_READ_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
#    define ETHR_HAVE_SU_DW_NATMC_READ_WB 1
#  endif
# 892 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_READ_ACQB
#  undef ETHR_HAVE_SU_DW_NATMC_READ_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_READ_ACQB 1
#  endif
# 897 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_READ_RELB
#  undef ETHR_HAVE_SU_DW_NATMC_READ_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
#    define ETHR_HAVE_SU_DW_NATMC_READ_RELB 1
#  endif
# 902 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_READ_MB
#  undef ETHR_HAVE_SU_DW_NATMC_READ_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
#    define ETHR_HAVE_SU_DW_NATMC_READ_MB 1
#  endif
# 907 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_INIT
#  undef ETHR_HAVE_SU_DW_NATMC_INIT
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
#    define ETHR_HAVE_SU_DW_NATMC_INIT 1
#  endif
# 912 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_INIT_RB
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_RB 1
#  endif
# 917 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_INIT_WB
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_WB 1
#  endif
# 922 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_INIT_ACQB
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_ACQB 1
#  endif
# 927 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_INIT_RELB
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_RELB 1
#  endif
# 932 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_DW_NATMC_INIT_MB
#  undef ETHR_HAVE_SU_DW_NATMC_INIT_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
#    define ETHR_HAVE_SU_DW_NATMC_INIT_MB 1
#  endif
# 937 "../include/internal/ethr_atomics.h"
#else
# 938 "../include/internal/ethr_atomics.h"
#  error "Invalid native atomic size"
#endif
# 940 "../include/internal/ethr_atomics.h"


#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)

#if (!defined(ETHR_HAVE_DW_NATMC_CMPXCHG) \
     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB) \
     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB) \
     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB) \
     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB) \
     && !defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB))
#  error "No native cmpxchg() op available"
#endif
# 952 "../include/internal/ethr_atomics.h"


/*
 * Read op used together with cmpxchg() fallback when no native op present.
 */
#if defined(ETHR_HAVE_DW_NATMC_READ)
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  ETHR_DW_NATMC_FUNC__(read)(VAR, VAL)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
# 961 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 964 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  ETHR_DW_NATMC_FUNC__(read_rb)(VAR, VAL)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 967 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 970 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  ETHR_DW_NATMC_FUNC__(read_wb)(VAR, VAL)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 973 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 976 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  ETHR_DW_NATMC_FUNC__(read_acqb)(VAR, VAL)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 979 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 982 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  ETHR_DW_NATMC_FUNC__(read_relb)(VAR, VAL)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 985 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 988 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  ETHR_DW_NATMC_FUNC__(read_mb)(VAR, VAL)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 991 "../include/internal/ethr_atomics.h"
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
  VAL.dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
#else
# 994 "../include/internal/ethr_atomics.h"
/*
 * We have no native read() op; guess zero and then use the
 * the atomics actual value returned from cmpxchg().
 */
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, VAL) \
do { \
  VAL.sint[0] = (ethr_sint_t) 0; \
  VAL.sint[1] = (ethr_sint_t) 0; \
} while (0)
#endif
# 1004 "../include/internal/ethr_atomics.h"

/*
 * Native cmpxchg() fallback used when no native op present.
 */
#define ETHR_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
do { \
    int res__; \
    ethr_dw_sint_t AVAL, exp_act__; \
    ETHR_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \
    do { \
	AVAL.sint[0] = exp_act__.sint[0]; \
	AVAL.sint[1] = exp_act__.sint[1]; \
	{ OPS; } \
	res__ = CMPXCHG(VAR, AVAL.sint, exp_act__.sint); \
    } while (__builtin_expect(res__ == 0, 0)); \
} while (0)


#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)
# 1023 "../include/internal/ethr_atomics.h"

#if (!defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG) \
     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB) \
     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB) \
     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB) \
     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB) \
     && !defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB))
#  error "No native cmpxchg() op available"
#endif
# 1032 "../include/internal/ethr_atomics.h"


/*
 * Read op used together with cmpxchg() fallback when no native op present.
 */
#if defined(ETHR_HAVE_SU_DW_NATMC_READ)
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_SU_DW_NATMC_FUNC__(read)(VAR)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 1041 "../include/internal/ethr_atomics.h"
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_SU_DW_NATMC_FUNC__(read_rb)(VAR)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 1044 "../include/internal/ethr_atomics.h"
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_SU_DW_NATMC_FUNC__(read_wb)(VAR)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 1047 "../include/internal/ethr_atomics.h"
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_SU_DW_NATMC_FUNC__(read_acqb)(VAR)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 1050 "../include/internal/ethr_atomics.h"
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_SU_DW_NATMC_FUNC__(read_relb)(VAR)
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 1053 "../include/internal/ethr_atomics.h"
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_SU_DW_NATMC_FUNC__(read_mb)(VAR)
#else
# 1056 "../include/internal/ethr_atomics.h"
/*
 * We have no native read() op; guess zero and then use the
 * the atomics actual value returned from cmpxchg().
 */
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ((ETHR_SU_DW_NAINT_T__) 0)
#endif
# 1063 "../include/internal/ethr_atomics.h"

/*
 * Native cmpxchg() fallback used when no native op present.
 */
#define ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
do { \
    ETHR_SU_DW_NAINT_T__ AVAL; \
    ETHR_SU_DW_NAINT_T__ new__, act__, exp__; \
    act__ = ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
    do { \
	exp__ = act__; \
	AVAL = (ETHR_SU_DW_NAINT_T__) act__; \
	{ OPS; } \
	new__ = (ETHR_SU_DW_NAINT_T__) AVAL; \
	act__ = CMPXCHG(VAR, new__, exp__); \
    } while (__builtin_expect(act__ != exp__, 0)); \
} while (0)


#else
# 1083 "../include/internal/ethr_atomics.h"
#  error "?!?"
#endif
# 1085 "../include/internal/ethr_atomics.h"



/* --- addr() --- */

static ETHR_INLINE ethr_sint_t *ETHR_DW_ATMC_FUNC__(addr)(ethr_dw_atomic_t *var)
{
#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1095 "../include/internal/ethr_atomics.h"

    return (ethr_sint_t *) ETHR_DW_NATMC_ADDR_FUNC__((&var->native));

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { return ETHR_DW_ATOMIC_FUNC__(addr)(var); }
#endif
# 1101 "../include/internal/ethr_atomics.h"

}


/* --- cmpxchg() --- */


static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1115 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1122 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1127 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1132 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1137 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1142 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1147 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1149 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1151 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1153 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1155 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1157 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
#else
# 1159 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_cmpxchg()!"
#endif
# 1161 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); }
#endif
# 1165 "../include/internal/ethr_atomics.h"

    return res;
}

static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1176 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1183 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1189 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1194 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1200 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1206 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1212 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1214 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1217 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1219 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1222 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1225 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 1228 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_cmpxchg_rb()!"
#endif
# 1230 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); }
#endif
# 1234 "../include/internal/ethr_atomics.h"

    return res;
}

static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1245 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1252 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_StoreStore);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1258 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1263 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_StoreStore);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1269 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_StoreStore);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1275 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_StoreStore);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1281 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1283 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1286 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1288 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1291 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1294 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
#else
# 1297 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_cmpxchg_wb()!"
#endif
# 1299 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(var, val, old_val); }
#endif
# 1303 "../include/internal/ethr_atomics.h"

    return res;
}

static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1314 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1321 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1327 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1333 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1338 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1344 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1350 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1352 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1355 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1358 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1360 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1363 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 1366 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_cmpxchg_acqb()!"
#endif
# 1368 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(var, val, old_val); }
#endif
# 1372 "../include/internal/ethr_atomics.h"

    return res;
}

static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1383 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1390 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1396 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1402 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1407 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1413 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1419 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1421 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1424 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1427 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1429 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1432 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
#else
# 1435 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_cmpxchg_relb()!"
#endif
# 1437 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(var, val, old_val); }
#endif
# 1441 "../include/internal/ethr_atomics.h"

    return res;
}

static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1452 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1459 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1465 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->dw_sint, old_val->dw_sint);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1471 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1478 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1485 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NAINT_T__ act;
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    act = ETHR_SU_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->dw_sint, old_val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    res = (act == old_val->dw_sint);
    old_val->dw_sint = act;
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1492 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_mb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1494 "../include/internal/ethr_atomics.h"
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_relb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1497 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_acqb)(&var->native, val->sint, old_val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1500 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_wb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1504 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg_rb)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1508 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = ETHR_DW_NATMC_FUNC__(cmpxchg)(&var->native, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 1512 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_cmpxchg_mb()!"
#endif
# 1514 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { res = ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(var, val, old_val); }
#endif
# 1518 "../include/internal/ethr_atomics.h"

    return res;
}

static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_DW_ATMC_FUNC__(cmpxchg)(var, val, old_val);
#else
# 1527 "../include/internal/ethr_atomics.h"
    return ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
#endif
# 1529 "../include/internal/ethr_atomics.h"
}


/* --- set() --- */


static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1541 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_SET)
    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
# 1545 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
# 1547 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
# 1549 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
# 1551 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
# 1553 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET)
# 1555 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
# 1557 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
# 1559 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
# 1561 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
# 1563 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
# 1565 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1567 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1569 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1571 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1573 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1575 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1577 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1579 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1581 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1583 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1585 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1587 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1589 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#else
# 1591 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_set()!"
#endif
# 1593 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(set)(var, val); }
#endif
# 1597 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1606 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
# 1610 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
# 1613 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
# 1615 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
# 1618 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
# 1621 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
# 1624 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET)
# 1626 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
# 1629 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
# 1631 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
# 1634 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
# 1637 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1640 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1642 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1645 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1647 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1650 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1653 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1656 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1658 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1661 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1663 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1666 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1669 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 1672 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_set_rb()!"
#endif
# 1674 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); }
#endif
# 1678 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1687 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
# 1691 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
# 1694 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
# 1696 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
# 1699 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
# 1702 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
# 1705 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET)
# 1707 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
# 1710 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
# 1712 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
# 1715 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
# 1718 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1721 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1723 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1726 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1728 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1731 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1734 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1737 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1739 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1742 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1744 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1747 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1750 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#else
# 1753 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_set_wb()!"
#endif
# 1755 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(set_wb)(var, val); }
#endif
# 1759 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1768 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
# 1772 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
# 1775 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
# 1778 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
# 1780 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
# 1783 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
# 1786 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
# 1788 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET)
# 1791 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
# 1794 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
# 1796 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
# 1799 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1802 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1804 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1807 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1810 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1812 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1815 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1818 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1820 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1823 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1826 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1828 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1831 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 1834 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_set_acqb()!"
#endif
# 1836 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(set_acqb)(var, val); }
#endif
# 1840 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1849 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
# 1853 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
# 1856 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
# 1859 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
# 1861 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
# 1864 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
# 1867 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
# 1869 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET)
# 1872 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
# 1875 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
# 1877 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
# 1880 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1883 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1885 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1888 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1891 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1893 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1896 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1899 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 1901 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 1904 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1907 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 1909 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1912 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#else
# 1915 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_set_relb()!"
#endif
# 1917 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(set_relb)(var, val); }
#endif
# 1921 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 1930 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_SET_MB)
    ETHR_SU_DW_NATMC_FUNC__(set_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RELB)
# 1934 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(set_relb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_ACQB)
# 1937 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_WB)
# 1940 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_SU_DW_NATMC_FUNC__(set_wb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET_RB)
# 1944 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set_rb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_SET)
# 1948 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(set)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET_MB)
# 1952 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RELB)
# 1954 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(set_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET_ACQB)
# 1957 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_SET_WB)
# 1960 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_DW_NATMC_FUNC__(set_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET_RB)
# 1964 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set_rb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_SET)
# 1968 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(set)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_MB)
# 1972 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RELB)
# 1974 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_ACQB)
# 1977 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval = val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_WB)
# 1980 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG_RB)
# 1984 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_CMPXCHG)
# 1988 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_SU_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval = val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_MB)
# 1992 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_mb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RELB)
# 1994 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_relb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_ACQB)
# 1997 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_acqb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_WB)
# 2000 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_wb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG_RB)
# 2004 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg_rb), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_CMPXCHG)
# 2008 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_CMPXCHG_FALLBACK__(ETHR_DW_NATMC_FUNC__(cmpxchg), &var->native, aval, aval.sint[0] = val->sint[0]; aval.sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 2012 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_dw_atomic_set_mb()!"
#endif
# 2014 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(set_mb)(var, val); }
#endif
# 2018 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_DW_ATMC_FUNC__(set)(var, val);
#else
# 2026 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
#endif
# 2028 "../include/internal/ethr_atomics.h"
}


/* --- read() --- */


static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2040 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_READ)
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 2044 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 2046 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 2048 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 2050 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 2052 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
#elif defined(ETHR_HAVE_DW_NATMC_READ)
# 2054 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 2056 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 2058 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 2060 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 2062 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 2064 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
#else
# 2066 "../include/internal/ethr_atomics.h"
    ethr_dw_sint_t tmp;
    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    (void) ETHR_DW_ATMC_FUNC__(cmpxchg)(var, &tmp, val);
#endif
# 2073 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(read)(var, val); }
#endif
# 2077 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2086 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
# 2090 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 2093 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 2095 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 2098 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 2101 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 2104 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ)
# 2106 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 2109 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 2111 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 2114 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 2117 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 2120 "../include/internal/ethr_atomics.h"
    ethr_dw_sint_t tmp;
    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, &tmp, val);
#endif
# 2127 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); }
#endif
# 2131 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2140 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
# 2144 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 2147 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 2149 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 2152 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 2155 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 2158 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ)
# 2160 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 2163 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 2165 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 2168 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 2171 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
#else
# 2174 "../include/internal/ethr_atomics.h"
    ethr_dw_sint_t tmp;
    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_wb)(var, &tmp, val);
#endif
# 2181 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(read_wb)(var, val); }
#endif
# 2185 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2194 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 2198 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
# 2201 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 2204 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 2206 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 2209 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 2212 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 2214 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ)
# 2217 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 2220 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 2222 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 2225 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
# 2228 "../include/internal/ethr_atomics.h"
    ethr_dw_sint_t tmp;
    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_acqb)(var, &tmp, val);
#endif
# 2235 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(read_acqb)(var, val); }
#endif
# 2239 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2248 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 2252 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
# 2255 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
# 2258 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 2260 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 2263 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 2266 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 2268 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ)
# 2271 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 2274 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 2276 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 2279 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
#else
# 2282 "../include/internal/ethr_atomics.h"
    ethr_dw_sint_t tmp;
    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_relb)(var, &tmp, val);
#endif
# 2289 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(read_relb)(var, val); }
#endif
# 2293 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2302 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_READ_MB)
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_mb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RELB)
# 2306 "../include/internal/ethr_atomics.h"
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_relb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_ACQB)
# 2309 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_acqb)(&var->native);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_WB)
# 2312 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_wb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ_RB)
# 2316 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read_rb)(&var->native);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_READ)
# 2320 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    val->dw_sint = ETHR_SU_DW_NATMC_FUNC__(read)(&var->native);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ_MB)
# 2324 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RELB)
# 2326 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(read_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ_ACQB)
# 2329 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_READ_WB)
# 2332 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ_RB)
# 2336 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read_rb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_DW_NATMC_READ)
# 2340 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(read)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
# 2344 "../include/internal/ethr_atomics.h"
    ethr_dw_sint_t tmp;
    tmp.sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    tmp.sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[0] = ETHR_UNUSUAL_SINT_VAL__;
    val->sint[1] = ETHR_UNUSUAL_SINT_VAL__;
    (void) ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(var, &tmp, val);
#endif
# 2351 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(read_mb)(var, val); }
#endif
# 2355 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_DW_ATMC_FUNC__(read)(var, val);
#else
# 2363 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(read_rb)(var, val);
#endif
# 2365 "../include/internal/ethr_atomics.h"
}


/* --- init() --- */


static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2377 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_INIT)
    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
# 2381 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
# 2383 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
# 2385 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
# 2387 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
# 2389 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT)
# 2391 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
# 2393 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
# 2395 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
# 2397 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
# 2399 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
# 2401 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
#else
# 2403 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set)(var, val);
#endif
# 2405 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(init)(var, val); }
#endif
# 2409 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2418 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
# 2422 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
# 2425 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
# 2427 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
# 2430 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
# 2433 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
# 2436 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT)
# 2438 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
# 2441 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
# 2443 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
# 2446 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
# 2449 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 2452 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
#endif
# 2454 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); }
#endif
# 2458 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2467 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
# 2471 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
# 2474 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
# 2476 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
# 2479 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
# 2482 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
# 2485 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT)
# 2487 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
# 2490 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
# 2492 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
# 2495 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
# 2498 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
#else
# 2501 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set_wb)(var, val);
#endif
# 2503 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(init_wb)(var, val); }
#endif
# 2507 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2516 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
# 2520 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
# 2523 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
# 2526 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
# 2528 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
# 2531 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
# 2534 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
# 2536 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT)
# 2539 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
# 2542 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
# 2544 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
# 2547 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 2550 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set_acqb)(var, val);
#endif
# 2552 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(init_acqb)(var, val); }
#endif
# 2556 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2565 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
# 2569 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
# 2572 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
# 2575 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
# 2577 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
# 2580 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
# 2583 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
# 2585 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT)
# 2588 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
# 2591 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
# 2593 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
# 2596 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
#else
# 2599 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set_relb)(var, val);
#endif
# 2601 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(init_relb)(var, val); }
#endif
# 2605 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__) {
#endif
# 2614 "../include/internal/ethr_atomics.h"

#if defined(ETHR_HAVE_SU_DW_NATMC_INIT_MB)
    ETHR_SU_DW_NATMC_FUNC__(init_mb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RELB)
# 2618 "../include/internal/ethr_atomics.h"
    ETHR_SU_DW_NATMC_FUNC__(init_relb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_ACQB)
# 2621 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_acqb)(&var->native, val->dw_sint);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_WB)
# 2624 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_SU_DW_NATMC_FUNC__(init_wb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT_RB)
# 2628 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init_rb)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_SU_DW_NATMC_INIT)
# 2632 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_SU_DW_NATMC_FUNC__(init)(&var->native, val->dw_sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_MB)
# 2636 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_mb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RELB)
# 2638 "../include/internal/ethr_atomics.h"
    ETHR_DW_NATMC_FUNC__(init_relb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_ACQB)
# 2641 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_acqb)(&var->native, val->sint);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_WB)
# 2644 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_DW_NATMC_FUNC__(init_wb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT_RB)
# 2648 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init_rb)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_DW_NATMC_INIT)
# 2652 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_DW_NATMC_FUNC__(init)(&var->native, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 2656 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(set_mb)(var, val);
#endif
# 2658 "../include/internal/ethr_atomics.h"

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    } else { ETHR_DW_ATOMIC_FUNC__(init_mb)(var, val); }
#endif
# 2662 "../include/internal/ethr_atomics.h"

}

static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_DW_ATMC_FUNC__(init)(var, val);
#else
# 2670 "../include/internal/ethr_atomics.h"
    ETHR_DW_ATMC_FUNC__(init_rb)(var, val);
#endif
# 2672 "../include/internal/ethr_atomics.h"
}

#endif /* ETHR_DW_ATMC_INLINE__ */
# 2675 "../include/internal/ethr_atomics.h"


/* ---------- Word size atomic implementation ---------- */


#ifdef ETHR_NEED_ATMC_PROTOTYPES__
ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var);
void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val);
void ethr_atomic_inc(ethr_atomic_t *var);
void ethr_atomic_inc_ddrb(ethr_atomic_t *var);
void ethr_atomic_inc_rb(ethr_atomic_t *var);
void ethr_atomic_inc_wb(ethr_atomic_t *var);
void ethr_atomic_inc_acqb(ethr_atomic_t *var);
void ethr_atomic_inc_relb(ethr_atomic_t *var);
void ethr_atomic_inc_mb(ethr_atomic_t *var);
void ethr_atomic_dec(ethr_atomic_t *var);
void ethr_atomic_dec_ddrb(ethr_atomic_t *var);
void ethr_atomic_dec_rb(ethr_atomic_t *var);
void ethr_atomic_dec_wb(ethr_atomic_t *var);
void ethr_atomic_dec_acqb(ethr_atomic_t *var);
void ethr_atomic_dec_relb(ethr_atomic_t *var);
void ethr_atomic_dec_mb(ethr_atomic_t *var);
ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val);
ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val);
#endif /* ETHR_NEED_ATMC_PROTOTYPES__ */
# 2774 "../include/internal/ethr_atomics.h"

#if (defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
     && (defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))

#if !defined(ETHR_NATMC_BITS__)
#  error "Missing native atomic implementation"
#elif ETHR_NATMC_BITS__ == 64
# 2781 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
#    define ETHR_HAVE_NATMC_CMPXCHG 1
#  endif
# 2785 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
#    define ETHR_HAVE_NATMC_CMPXCHG_RB 1
#  endif
# 2789 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
#    define ETHR_HAVE_NATMC_CMPXCHG_WB 1
#  endif
# 2793 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
#    define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
#  endif
# 2797 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
#    define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
#  endif
# 2801 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
#    define ETHR_HAVE_NATMC_CMPXCHG_MB 1
#  endif
# 2805 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
#    define ETHR_HAVE_NATMC_XCHG 1
#  endif
# 2809 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
#    define ETHR_HAVE_NATMC_XCHG_RB 1
#  endif
# 2813 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
#    define ETHR_HAVE_NATMC_XCHG_WB 1
#  endif
# 2817 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
#    define ETHR_HAVE_NATMC_XCHG_ACQB 1
#  endif
# 2821 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
#    define ETHR_HAVE_NATMC_XCHG_RELB 1
#  endif
# 2825 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
#    define ETHR_HAVE_NATMC_XCHG_MB 1
#  endif
# 2829 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
#    define ETHR_HAVE_NATMC_SET 1
#  endif
# 2833 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
#    define ETHR_HAVE_NATMC_SET_RB 1
#  endif
# 2837 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
#    define ETHR_HAVE_NATMC_SET_WB 1
#  endif
# 2841 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
#    define ETHR_HAVE_NATMC_SET_ACQB 1
#  endif
# 2845 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
#    define ETHR_HAVE_NATMC_SET_RELB 1
#  endif
# 2849 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
#    define ETHR_HAVE_NATMC_SET_MB 1
#  endif
# 2853 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
#    define ETHR_HAVE_NATMC_INIT 1
#  endif
# 2857 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
#    define ETHR_HAVE_NATMC_INIT_RB 1
#  endif
# 2861 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
#    define ETHR_HAVE_NATMC_INIT_WB 1
#  endif
# 2865 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
#    define ETHR_HAVE_NATMC_INIT_ACQB 1
#  endif
# 2869 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
#    define ETHR_HAVE_NATMC_INIT_RELB 1
#  endif
# 2873 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
#    define ETHR_HAVE_NATMC_INIT_MB 1
#  endif
# 2877 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
#    define ETHR_HAVE_NATMC_ADD_RETURN 1
#  endif
# 2881 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
#    define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
#  endif
# 2885 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
#    define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
#  endif
# 2889 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
#    define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
#  endif
# 2893 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
#    define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
#  endif
# 2897 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
#    define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
#  endif
# 2901 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
#    define ETHR_HAVE_NATMC_READ 1
#  endif
# 2905 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
#    define ETHR_HAVE_NATMC_READ_RB 1
#  endif
# 2909 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
#    define ETHR_HAVE_NATMC_READ_WB 1
#  endif
# 2913 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
#    define ETHR_HAVE_NATMC_READ_ACQB 1
#  endif
# 2917 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
#    define ETHR_HAVE_NATMC_READ_RELB 1
#  endif
# 2921 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
#    define ETHR_HAVE_NATMC_READ_MB 1
#  endif
# 2925 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
#    define ETHR_HAVE_NATMC_INC_RETURN 1
#  endif
# 2929 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
#    define ETHR_HAVE_NATMC_INC_RETURN_RB 1
#  endif
# 2933 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
#    define ETHR_HAVE_NATMC_INC_RETURN_WB 1
#  endif
# 2937 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
#    define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
#  endif
# 2941 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
#    define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
#  endif
# 2945 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
#    define ETHR_HAVE_NATMC_INC_RETURN_MB 1
#  endif
# 2949 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
#    define ETHR_HAVE_NATMC_DEC_RETURN 1
#  endif
# 2953 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
#    define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
#  endif
# 2957 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
#    define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
#  endif
# 2961 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
#    define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
#  endif
# 2965 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
#    define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
#  endif
# 2969 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
#    define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
#  endif
# 2973 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
#    define ETHR_HAVE_NATMC_ADD 1
#  endif
# 2977 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
#    define ETHR_HAVE_NATMC_ADD_RB 1
#  endif
# 2981 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
#    define ETHR_HAVE_NATMC_ADD_WB 1
#  endif
# 2985 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
#    define ETHR_HAVE_NATMC_ADD_ACQB 1
#  endif
# 2989 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
#    define ETHR_HAVE_NATMC_ADD_RELB 1
#  endif
# 2993 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
#    define ETHR_HAVE_NATMC_ADD_MB 1
#  endif
# 2997 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
#    define ETHR_HAVE_NATMC_INC 1
#  endif
# 3001 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
#    define ETHR_HAVE_NATMC_INC_RB 1
#  endif
# 3005 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
#    define ETHR_HAVE_NATMC_INC_WB 1
#  endif
# 3009 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
#    define ETHR_HAVE_NATMC_INC_ACQB 1
#  endif
# 3013 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
#    define ETHR_HAVE_NATMC_INC_RELB 1
#  endif
# 3017 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
#    define ETHR_HAVE_NATMC_INC_MB 1
#  endif
# 3021 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
#    define ETHR_HAVE_NATMC_DEC 1
#  endif
# 3025 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
#    define ETHR_HAVE_NATMC_DEC_RB 1
#  endif
# 3029 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
#    define ETHR_HAVE_NATMC_DEC_WB 1
#  endif
# 3033 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
#    define ETHR_HAVE_NATMC_DEC_ACQB 1
#  endif
# 3037 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
#    define ETHR_HAVE_NATMC_DEC_RELB 1
#  endif
# 3041 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
#    define ETHR_HAVE_NATMC_DEC_MB 1
#  endif
# 3045 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
#    define ETHR_HAVE_NATMC_AND_RETOLD 1
#  endif
# 3049 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
#    define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
#  endif
# 3053 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
#    define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
#  endif
# 3057 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
#    define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
#  endif
# 3061 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
#    define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
#  endif
# 3065 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
#    define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
#  endif
# 3069 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
#    define ETHR_HAVE_NATMC_OR_RETOLD 1
#  endif
# 3073 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
#    define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
#  endif
# 3077 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
#    define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
#  endif
# 3081 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
#    define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
#  endif
# 3085 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
#    define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
#  endif
# 3089 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
#    define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
#  endif
# 3093 "../include/internal/ethr_atomics.h"
#elif ETHR_NATMC_BITS__ == 32
# 3094 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
#    define ETHR_HAVE_NATMC_CMPXCHG 1
#  endif
# 3098 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
#    define ETHR_HAVE_NATMC_CMPXCHG_RB 1
#  endif
# 3102 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
#    define ETHR_HAVE_NATMC_CMPXCHG_WB 1
#  endif
# 3106 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
#    define ETHR_HAVE_NATMC_CMPXCHG_ACQB 1
#  endif
# 3110 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
#    define ETHR_HAVE_NATMC_CMPXCHG_RELB 1
#  endif
# 3114 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_CMPXCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
#    define ETHR_HAVE_NATMC_CMPXCHG_MB 1
#  endif
# 3118 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
#    define ETHR_HAVE_NATMC_XCHG 1
#  endif
# 3122 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
#    define ETHR_HAVE_NATMC_XCHG_RB 1
#  endif
# 3126 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
#    define ETHR_HAVE_NATMC_XCHG_WB 1
#  endif
# 3130 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
#    define ETHR_HAVE_NATMC_XCHG_ACQB 1
#  endif
# 3134 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
#    define ETHR_HAVE_NATMC_XCHG_RELB 1
#  endif
# 3138 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_XCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
#    define ETHR_HAVE_NATMC_XCHG_MB 1
#  endif
# 3142 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
#    define ETHR_HAVE_NATMC_SET 1
#  endif
# 3146 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
#    define ETHR_HAVE_NATMC_SET_RB 1
#  endif
# 3150 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
#    define ETHR_HAVE_NATMC_SET_WB 1
#  endif
# 3154 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
#    define ETHR_HAVE_NATMC_SET_ACQB 1
#  endif
# 3158 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
#    define ETHR_HAVE_NATMC_SET_RELB 1
#  endif
# 3162 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_SET_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
#    define ETHR_HAVE_NATMC_SET_MB 1
#  endif
# 3166 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
#    define ETHR_HAVE_NATMC_INIT 1
#  endif
# 3170 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
#    define ETHR_HAVE_NATMC_INIT_RB 1
#  endif
# 3174 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
#    define ETHR_HAVE_NATMC_INIT_WB 1
#  endif
# 3178 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
#    define ETHR_HAVE_NATMC_INIT_ACQB 1
#  endif
# 3182 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
#    define ETHR_HAVE_NATMC_INIT_RELB 1
#  endif
# 3186 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INIT_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
#    define ETHR_HAVE_NATMC_INIT_MB 1
#  endif
# 3190 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
#    define ETHR_HAVE_NATMC_ADD_RETURN 1
#  endif
# 3194 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
#    define ETHR_HAVE_NATMC_ADD_RETURN_RB 1
#  endif
# 3198 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
#    define ETHR_HAVE_NATMC_ADD_RETURN_WB 1
#  endif
# 3202 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
#    define ETHR_HAVE_NATMC_ADD_RETURN_ACQB 1
#  endif
# 3206 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
#    define ETHR_HAVE_NATMC_ADD_RETURN_RELB 1
#  endif
# 3210 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
#    define ETHR_HAVE_NATMC_ADD_RETURN_MB 1
#  endif
# 3214 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
#    define ETHR_HAVE_NATMC_READ 1
#  endif
# 3218 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
#    define ETHR_HAVE_NATMC_READ_RB 1
#  endif
# 3222 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
#    define ETHR_HAVE_NATMC_READ_WB 1
#  endif
# 3226 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
#    define ETHR_HAVE_NATMC_READ_ACQB 1
#  endif
# 3230 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
#    define ETHR_HAVE_NATMC_READ_RELB 1
#  endif
# 3234 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_READ_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
#    define ETHR_HAVE_NATMC_READ_MB 1
#  endif
# 3238 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
#    define ETHR_HAVE_NATMC_INC_RETURN 1
#  endif
# 3242 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
#    define ETHR_HAVE_NATMC_INC_RETURN_RB 1
#  endif
# 3246 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
#    define ETHR_HAVE_NATMC_INC_RETURN_WB 1
#  endif
# 3250 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
#    define ETHR_HAVE_NATMC_INC_RETURN_ACQB 1
#  endif
# 3254 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
#    define ETHR_HAVE_NATMC_INC_RETURN_RELB 1
#  endif
# 3258 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
#    define ETHR_HAVE_NATMC_INC_RETURN_MB 1
#  endif
# 3262 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
#    define ETHR_HAVE_NATMC_DEC_RETURN 1
#  endif
# 3266 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
#    define ETHR_HAVE_NATMC_DEC_RETURN_RB 1
#  endif
# 3270 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
#    define ETHR_HAVE_NATMC_DEC_RETURN_WB 1
#  endif
# 3274 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
#    define ETHR_HAVE_NATMC_DEC_RETURN_ACQB 1
#  endif
# 3278 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
#    define ETHR_HAVE_NATMC_DEC_RETURN_RELB 1
#  endif
# 3282 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
#    define ETHR_HAVE_NATMC_DEC_RETURN_MB 1
#  endif
# 3286 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
#    define ETHR_HAVE_NATMC_ADD 1
#  endif
# 3290 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
#    define ETHR_HAVE_NATMC_ADD_RB 1
#  endif
# 3294 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
#    define ETHR_HAVE_NATMC_ADD_WB 1
#  endif
# 3298 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
#    define ETHR_HAVE_NATMC_ADD_ACQB 1
#  endif
# 3302 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
#    define ETHR_HAVE_NATMC_ADD_RELB 1
#  endif
# 3306 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_ADD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
#    define ETHR_HAVE_NATMC_ADD_MB 1
#  endif
# 3310 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
#    define ETHR_HAVE_NATMC_INC 1
#  endif
# 3314 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
#    define ETHR_HAVE_NATMC_INC_RB 1
#  endif
# 3318 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
#    define ETHR_HAVE_NATMC_INC_WB 1
#  endif
# 3322 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
#    define ETHR_HAVE_NATMC_INC_ACQB 1
#  endif
# 3326 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
#    define ETHR_HAVE_NATMC_INC_RELB 1
#  endif
# 3330 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_INC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
#    define ETHR_HAVE_NATMC_INC_MB 1
#  endif
# 3334 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
#    define ETHR_HAVE_NATMC_DEC 1
#  endif
# 3338 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
#    define ETHR_HAVE_NATMC_DEC_RB 1
#  endif
# 3342 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
#    define ETHR_HAVE_NATMC_DEC_WB 1
#  endif
# 3346 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
#    define ETHR_HAVE_NATMC_DEC_ACQB 1
#  endif
# 3350 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
#    define ETHR_HAVE_NATMC_DEC_RELB 1
#  endif
# 3354 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_DEC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
#    define ETHR_HAVE_NATMC_DEC_MB 1
#  endif
# 3358 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
#    define ETHR_HAVE_NATMC_AND_RETOLD 1
#  endif
# 3362 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
#    define ETHR_HAVE_NATMC_AND_RETOLD_RB 1
#  endif
# 3366 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
#    define ETHR_HAVE_NATMC_AND_RETOLD_WB 1
#  endif
# 3370 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
#    define ETHR_HAVE_NATMC_AND_RETOLD_ACQB 1
#  endif
# 3374 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
#    define ETHR_HAVE_NATMC_AND_RETOLD_RELB 1
#  endif
# 3378 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_AND_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
#    define ETHR_HAVE_NATMC_AND_RETOLD_MB 1
#  endif
# 3382 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
#    define ETHR_HAVE_NATMC_OR_RETOLD 1
#  endif
# 3386 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
#    define ETHR_HAVE_NATMC_OR_RETOLD_RB 1
#  endif
# 3390 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
#    define ETHR_HAVE_NATMC_OR_RETOLD_WB 1
#  endif
# 3394 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
#    define ETHR_HAVE_NATMC_OR_RETOLD_ACQB 1
#  endif
# 3398 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
#    define ETHR_HAVE_NATMC_OR_RETOLD_RELB 1
#  endif
# 3402 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC_OR_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
#    define ETHR_HAVE_NATMC_OR_RETOLD_MB 1
#  endif
# 3406 "../include/internal/ethr_atomics.h"
#else
# 3407 "../include/internal/ethr_atomics.h"
#  error "Invalid native atomic size"
#endif
# 3409 "../include/internal/ethr_atomics.h"

#if (!defined(ETHR_HAVE_NATMC_CMPXCHG) \
     && !defined(ETHR_HAVE_NATMC_CMPXCHG_RB) \
     && !defined(ETHR_HAVE_NATMC_CMPXCHG_WB) \
     && !defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB) \
     && !defined(ETHR_HAVE_NATMC_CMPXCHG_RELB) \
     && !defined(ETHR_HAVE_NATMC_CMPXCHG_MB))
#  error "No native cmpxchg() op available"
#endif
# 3418 "../include/internal/ethr_atomics.h"


/*
 * Read op used together with cmpxchg() fallback when no native op present.
 */
#if defined(ETHR_HAVE_NATMC_READ)
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC_FUNC__(read)(VAR)
#elif defined(ETHR_HAVE_NATMC_READ_RB)
# 3427 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC_FUNC__(read_rb)(VAR)
#elif defined(ETHR_HAVE_NATMC_READ_WB)
# 3430 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC_FUNC__(read_wb)(VAR)
#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
# 3433 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC_FUNC__(read_acqb)(VAR)
#elif defined(ETHR_HAVE_NATMC_READ_RELB)
# 3436 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC_FUNC__(read_relb)(VAR)
#elif defined(ETHR_HAVE_NATMC_READ_MB)
# 3439 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC_FUNC__(read_mb)(VAR)
#else
# 3442 "../include/internal/ethr_atomics.h"
/*
 * We have no native read() op; guess zero and then use the
 * the atomics actual value returned from cmpxchg().
 */
#define ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR) \
  ((ETHR_NAINT_T__) 0)
#endif
# 3449 "../include/internal/ethr_atomics.h"

/*
 * Native cmpxchg() fallback used when no native op present.
 */
#define ETHR_NATMC_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
do { \
    ethr_sint_t AVAL; \
    ETHR_NAINT_T__ new__, act__, exp__; \
    act__ = ETHR_NATMC_CMPXCHG_FALLBACK_READ__(VAR); \
    do { \
	exp__ = act__; \
	AVAL = (ethr_sint_t) act__; \
	{ OPS; } \
	new__ = (ETHR_NAINT_T__) AVAL; \
	act__ = CMPXCHG(VAR, new__, exp__); \
    } while (__builtin_expect(act__ != exp__, 0)); \
} while (0)



/* --- addr() --- */

static ETHR_INLINE ethr_sint_t *ETHR_ATMC_FUNC__(addr)(ethr_atomic_t *var)
{
    return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var);

}


/* --- cmpxchg() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_CMPXCHG)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3487 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3489 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3491 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3493 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3495 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#else
# 3497 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_cmpxchg()!"
#endif
# 3499 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_rb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3508 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3511 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3513 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3516 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3519 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 3522 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_cmpxchg_rb()!"
#endif
# 3524 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_wb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3533 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3536 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3538 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3541 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3544 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#else
# 3547 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_cmpxchg_wb()!"
#endif
# 3549 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_acqb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3558 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3561 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3564 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3566 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3569 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 3572 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_cmpxchg_acqb()!"
#endif
# 3574 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_relb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3583 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3586 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3589 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3591 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3594 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#else
# 3597 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_cmpxchg_relb()!"
#endif
# 3599 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3608 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3611 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3614 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3618 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3622 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, (ETHR_NAINT_T__) val, (ETHR_NAINT_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 3626 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_cmpxchg_mb()!"
#endif
# 3628 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(cmpxchg)(var, val, old_val);
#else
# 3636 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
#endif
# 3638 "../include/internal/ethr_atomics.h"
}


/* --- xchg() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_XCHG)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
# 3650 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
# 3652 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
# 3654 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
# 3656 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
# 3658 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3660 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3662 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3664 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3666 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3668 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3670 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#else
# 3672 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_xchg()!"
#endif
# 3674 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_XCHG_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG)
# 3683 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
# 3686 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
# 3688 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
# 3691 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
# 3694 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3697 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3699 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3702 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3704 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3707 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3710 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 3713 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_xchg_rb()!"
#endif
# 3715 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_XCHG_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG)
# 3724 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
# 3727 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
# 3729 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
# 3732 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
# 3735 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3738 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3740 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3743 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3745 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3748 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3751 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
#else
# 3754 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_xchg_wb()!"
#endif
# 3756 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_XCHG_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
# 3765 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_XCHG)
# 3768 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
# 3771 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
# 3773 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
# 3776 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3779 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3781 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3784 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3787 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3789 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3792 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 3795 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_xchg_acqb()!"
#endif
# 3797 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_XCHG_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
# 3806 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG)
# 3809 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_MB)
# 3812 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
# 3814 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
# 3817 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3820 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3822 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3825 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3828 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3830 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3833 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#else
# 3836 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_xchg_relb()!"
#endif
# 3838 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_XCHG_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_RELB)
# 3847 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_XCHG_ACQB)
# 3850 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_XCHG_WB)
# 3853 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_XCHG_RB)
# 3857 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_XCHG)
# 3861 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 3865 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 3867 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 3870 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 3873 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 3877 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 3881 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 3885 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_xchg_mb()!"
#endif
# 3887 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(xchg)(var, val);
#else
# 3895 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(xchg_rb)(var, val);
#endif
# 3897 "../include/internal/ethr_atomics.h"
}


/* --- set() --- */


static ETHR_INLINE void ETHR_ATMC_FUNC__(set)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_SET)
    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RB)
# 3908 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_WB)
# 3910 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
# 3912 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RELB)
# 3914 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_MB)
# 3916 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
#else
# 3918 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(xchg)(var, val);
#endif
# 3920 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(set_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_SET_RB)
    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET)
# 3927 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_SET_MB)
# 3930 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_WB)
# 3932 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
# 3935 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_SET_RELB)
# 3938 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 3941 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(xchg_rb)(var, val);
#endif
# 3943 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(set_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_SET_WB)
    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET)
# 3950 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_MB)
# 3953 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RB)
# 3955 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
# 3958 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RELB)
# 3961 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
#else
# 3964 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(xchg_wb)(var, val);
#endif
# 3966 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(set_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_SET_ACQB)
    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RB)
# 3973 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_SET)
# 3976 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_SET_MB)
# 3979 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_WB)
# 3981 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_SET_RELB)
# 3984 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
# 3987 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(xchg_acqb)(var, val);
#endif
# 3989 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(set_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_SET_RELB)
    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_WB)
# 3996 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET)
# 3999 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_MB)
# 4002 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RB)
# 4004 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
# 4007 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
#else
# 4010 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(xchg_relb)(var, val);
#endif
# 4012 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_SET_MB)
    ETHR_NATMC_FUNC__(set_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_RELB)
# 4019 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_SET_ACQB)
# 4022 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_SET_WB)
# 4025 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_NATMC_FUNC__(set_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_SET_RB)
# 4029 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_SET)
# 4033 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4037 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(xchg_mb)(var, val);
#endif
# 4039 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(set_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC_FUNC__(set)(var, val);
#else
# 4046 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set_rb)(var, val);
#endif
# 4048 "../include/internal/ethr_atomics.h"
}


/* --- init() --- */


static ETHR_INLINE void ETHR_ATMC_FUNC__(init)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_INIT)
    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RB)
# 4059 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_WB)
# 4061 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
# 4063 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
# 4065 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_MB)
# 4067 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
#else
# 4069 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set)(var, val);
#endif
# 4071 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(init_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_INIT_RB)
    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT)
# 4078 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INIT_MB)
# 4081 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_WB)
# 4083 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
# 4086 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
# 4089 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 4092 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set_rb)(var, val);
#endif
# 4094 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(init_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_INIT_WB)
    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT)
# 4101 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_MB)
# 4104 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RB)
# 4106 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
# 4109 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
# 4112 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
#else
# 4115 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set_wb)(var, val);
#endif
# 4117 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(init_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_INIT_ACQB)
    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RB)
# 4124 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INIT)
# 4127 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INIT_MB)
# 4130 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_WB)
# 4132 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
# 4135 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4138 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set_acqb)(var, val);
#endif
# 4140 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(init_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_INIT_RELB)
    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_WB)
# 4147 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT)
# 4150 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_MB)
# 4153 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RB)
# 4155 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
# 4158 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
#else
# 4161 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set_relb)(var, val);
#endif
# 4163 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_INIT_MB)
    ETHR_NATMC_FUNC__(init_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_RELB)
# 4170 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(init_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INIT_ACQB)
# 4173 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_INIT_WB)
# 4176 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(init_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INIT_RB)
# 4180 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INIT)
# 4184 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4188 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(set_mb)(var, val);
#endif
# 4190 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(init_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC_FUNC__(init)(var, val);
#else
# 4197 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(init_rb)(var, val);
#endif
# 4199 "../include/internal/ethr_atomics.h"
}


/* --- add_read() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_ADD_RETURN)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
# 4211 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
# 4213 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
# 4215 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
# 4217 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
# 4219 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 4221 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 4223 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 4225 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 4227 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 4229 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 4231 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#else
# 4233 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_add_read()!"
#endif
# 4235 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
# 4244 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
# 4247 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
# 4249 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
# 4252 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
# 4255 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 4258 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 4260 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 4263 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 4265 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 4268 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 4271 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 4274 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_add_read_rb()!"
#endif
# 4276 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
# 4285 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
# 4288 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
# 4290 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
# 4293 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
# 4296 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 4299 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 4301 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 4304 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 4306 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 4309 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 4312 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#else
# 4315 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_add_read_wb()!"
#endif
# 4317 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
# 4326 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
# 4329 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
# 4332 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
# 4334 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
# 4337 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 4340 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 4342 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 4345 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 4348 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 4350 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 4353 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4356 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_add_read_acqb()!"
#endif
# 4358 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
# 4367 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
# 4370 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
# 4373 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
# 4375 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
# 4378 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 4381 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 4383 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 4386 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 4389 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 4391 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 4394 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#else
# 4397 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_add_read_relb()!"
#endif
# 4399 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_ADD_RETURN_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RELB)
# 4408 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_ACQB)
# 4411 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_WB)
# 4414 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN_RB)
# 4418 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RETURN)
# 4422 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 4426 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 4428 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 4431 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 4434 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 4438 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 4442 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4446 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_add_read_mb()!"
#endif
# 4448 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(add_read)(var, val);
#else
# 4456 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(add_read_rb)(var, val);
#endif
# 4458 "../include/internal/ethr_atomics.h"
}


/* --- read() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_READ)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RB)
# 4470 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_WB)
# 4472 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
# 4474 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RELB)
# 4476 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_MB)
# 4478 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
#else
# 4480 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(cmpxchg)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
# 4482 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_rb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_READ_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC_READ)
# 4491 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_READ_MB)
# 4494 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_WB)
# 4496 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
# 4499 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_READ_RELB)
# 4502 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 4505 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(cmpxchg_rb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
# 4507 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_wb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_READ_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
#elif defined(ETHR_HAVE_NATMC_READ)
# 4516 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
#elif defined(ETHR_HAVE_NATMC_READ_MB)
# 4519 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RB)
# 4521 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
# 4524 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RELB)
# 4527 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
#else
# 4530 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(cmpxchg_wb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
# 4532 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_acqb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_READ_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RB)
# 4541 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC_READ)
# 4544 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC_READ_MB)
# 4547 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_WB)
# 4549 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC_READ_RELB)
# 4552 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
# 4555 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(cmpxchg_acqb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
# 4557 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_relb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_READ_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_WB)
# 4566 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
#elif defined(ETHR_HAVE_NATMC_READ)
# 4569 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
#elif defined(ETHR_HAVE_NATMC_READ_MB)
# 4572 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RB)
# 4574 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
# 4577 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
#else
# 4580 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(cmpxchg_relb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
# 4582 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_READ_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_RELB)
# 4591 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC_READ_ACQB)
# 4594 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_READ_WB)
# 4597 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC_READ_RB)
# 4601 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC_READ)
# 4605 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
# 4609 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(cmpxchg_mb)(var, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__, (ethr_sint_t) ETHR_UNUSUAL_SINT_VAL__);
#endif
# 4611 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_ddrb)(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(read)(var);
#else
# 4619 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(read_rb)(var);
#endif
# 4621 "../include/internal/ethr_atomics.h"
}


/* --- inc_read() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_INC_RETURN)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
# 4633 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
# 4635 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
# 4637 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
# 4639 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
# 4641 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
#else
# 4643 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) 1);
#endif
# 4645 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_rb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
# 4654 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
# 4657 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
# 4659 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
# 4662 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
# 4665 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 4668 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) 1);
#endif
# 4670 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_wb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
# 4679 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
# 4682 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
# 4684 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
# 4687 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
# 4690 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
#else
# 4693 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) 1);
#endif
# 4695 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_acqb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
# 4704 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
# 4707 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
# 4710 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
# 4712 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
# 4715 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4718 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) 1);
#endif
# 4720 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_relb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
# 4729 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
# 4732 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
# 4735 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
# 4737 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
# 4740 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
#else
# 4743 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) 1);
#endif
# 4745 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_INC_RETURN_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RELB)
# 4754 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_ACQB)
# 4757 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_WB)
# 4760 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN_RB)
# 4764 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RETURN)
# 4768 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4772 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) 1);
#endif
# 4774 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_ddrb)(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(inc_read)(var);
#else
# 4782 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(inc_read_rb)(var);
#endif
# 4784 "../include/internal/ethr_atomics.h"
}


/* --- dec_read() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_DEC_RETURN)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
# 4796 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
# 4798 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
# 4800 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
# 4802 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
# 4804 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#else
# 4806 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read)(var, (ethr_sint_t) -1);
#endif
# 4808 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_rb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
# 4817 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
# 4820 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
# 4822 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
# 4825 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
# 4828 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 4831 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_rb)(var, (ethr_sint_t) -1);
#endif
# 4833 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_wb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
# 4842 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
# 4845 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
# 4847 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
# 4850 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
# 4853 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
#else
# 4856 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_wb)(var, (ethr_sint_t) -1);
#endif
# 4858 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_acqb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
# 4867 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
# 4870 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
# 4873 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
# 4875 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
# 4878 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4881 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_acqb)(var, (ethr_sint_t) -1);
#endif
# 4883 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_relb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
# 4892 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
# 4895 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
# 4898 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
# 4900 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
# 4903 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
#else
# 4906 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_relb)(var, (ethr_sint_t) -1);
#endif
# 4908 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_DEC_RETURN_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RELB)
# 4917 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_ACQB)
# 4920 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_WB)
# 4923 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN_RB)
# 4927 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RETURN)
# 4931 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 4935 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC_FUNC__(add_read_mb)(var, (ethr_sint_t) -1);
#endif
# 4937 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_ddrb)(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(dec_read)(var);
#else
# 4945 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(dec_read_rb)(var);
#endif
# 4947 "../include/internal/ethr_atomics.h"
}


/* --- add() --- */


static ETHR_INLINE void ETHR_ATMC_FUNC__(add)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_ADD)
    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RB)
# 4958 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_WB)
# 4960 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
# 4962 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
# 4964 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_MB)
# 4966 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#else
# 4968 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(add_read)(var, val);
#endif
# 4970 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(add_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_ADD_RB)
    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD)
# 4977 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_ADD_MB)
# 4980 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_WB)
# 4982 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
# 4985 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
# 4988 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 4991 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(add_read_rb)(var, val);
#endif
# 4993 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(add_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_ADD_WB)
    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD)
# 5000 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_MB)
# 5003 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RB)
# 5005 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
# 5008 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
# 5011 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
#else
# 5014 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(add_read_wb)(var, val);
#endif
# 5016 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(add_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_ADD_ACQB)
    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RB)
# 5023 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD)
# 5026 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_MB)
# 5029 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_WB)
# 5031 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
# 5034 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5037 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(add_read_acqb)(var, val);
#endif
# 5039 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(add_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_ADD_RELB)
    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_WB)
# 5046 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD)
# 5049 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_MB)
# 5052 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RB)
# 5054 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
# 5057 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
#else
# 5060 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(add_read_relb)(var, val);
#endif
# 5062 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
#if defined(ETHR_HAVE_NATMC_ADD_MB)
    ETHR_NATMC_FUNC__(add_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_RELB)
# 5069 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(add_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_ACQB)
# 5072 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_ADD_WB)
# 5075 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD_RB)
# 5079 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_ADD)
# 5083 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5087 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(add_read_mb)(var, val);
#endif
# 5089 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(add_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC_FUNC__(add)(var, val);
#else
# 5096 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(add_rb)(var, val);
#endif
# 5098 "../include/internal/ethr_atomics.h"
}


/* --- inc() --- */


static ETHR_INLINE void ETHR_ATMC_FUNC__(inc)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_INC)
    ETHR_NATMC_FUNC__(inc)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RB)
# 5109 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_WB)
# 5111 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_wb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
# 5113 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RELB)
# 5115 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_relb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_MB)
# 5117 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_mb)(var);
#else
# 5119 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(inc_read)(var);
#endif
# 5121 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_rb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_INC_RB)
    ETHR_NATMC_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC)
# 5128 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INC_MB)
# 5131 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_WB)
# 5133 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
# 5136 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_INC_RELB)
# 5139 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 5142 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(inc_read_rb)(var);
#endif
# 5144 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_wb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_INC_WB)
    ETHR_NATMC_FUNC__(inc_wb)(var);
#elif defined(ETHR_HAVE_NATMC_INC)
# 5151 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(inc)(var);
#elif defined(ETHR_HAVE_NATMC_INC_MB)
# 5154 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RB)
# 5156 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
# 5159 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RELB)
# 5162 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(inc_relb)(var);
#else
# 5165 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(inc_read_wb)(var);
#endif
# 5167 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_acqb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_INC_ACQB)
    ETHR_NATMC_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RB)
# 5174 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC)
# 5177 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_MB)
# 5180 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_WB)
# 5182 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RELB)
# 5185 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5188 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(inc_read_acqb)(var);
#endif
# 5190 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_relb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_INC_RELB)
    ETHR_NATMC_FUNC__(inc_relb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_WB)
# 5197 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc_wb)(var);
#elif defined(ETHR_HAVE_NATMC_INC)
# 5200 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc)(var);
#elif defined(ETHR_HAVE_NATMC_INC_MB)
# 5203 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RB)
# 5205 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
# 5208 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc_acqb)(var);
#else
# 5211 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(inc_read_relb)(var);
#endif
# 5213 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_INC_MB)
    ETHR_NATMC_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_RELB)
# 5220 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(inc_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_ACQB)
# 5223 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_INC_WB)
# 5226 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC_RB)
# 5230 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_INC)
# 5234 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(inc)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5238 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(inc_read_mb)(var);
#endif
# 5240 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_ddrb)(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC_FUNC__(inc)(var);
#else
# 5247 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(inc_rb)(var);
#endif
# 5249 "../include/internal/ethr_atomics.h"
}


/* --- dec() --- */


static ETHR_INLINE void ETHR_ATMC_FUNC__(dec)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_DEC)
    ETHR_NATMC_FUNC__(dec)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RB)
# 5260 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_WB)
# 5262 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_wb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
# 5264 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
# 5266 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_relb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_MB)
# 5268 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_mb)(var);
#else
# 5270 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(dec_read)(var);
#endif
# 5272 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_rb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_DEC_RB)
    ETHR_NATMC_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC)
# 5279 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_DEC_MB)
# 5282 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_WB)
# 5284 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
# 5287 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
# 5290 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 5293 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(dec_read_rb)(var);
#endif
# 5295 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_wb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_DEC_WB)
    ETHR_NATMC_FUNC__(dec_wb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC)
# 5302 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(dec)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_MB)
# 5305 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RB)
# 5307 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
# 5310 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
# 5313 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_FUNC__(dec_relb)(var);
#else
# 5316 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(dec_read_wb)(var);
#endif
# 5318 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_acqb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_DEC_ACQB)
    ETHR_NATMC_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RB)
# 5325 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC)
# 5328 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_MB)
# 5331 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_WB)
# 5333 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
# 5336 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5339 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(dec_read_acqb)(var);
#endif
# 5341 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_relb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_DEC_RELB)
    ETHR_NATMC_FUNC__(dec_relb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_WB)
# 5348 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec_wb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC)
# 5351 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_MB)
# 5354 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RB)
# 5356 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
# 5359 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec_acqb)(var);
#else
# 5362 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(dec_read_relb)(var);
#endif
# 5364 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var)
{
#if defined(ETHR_HAVE_NATMC_DEC_MB)
    ETHR_NATMC_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_RELB)
# 5371 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_FUNC__(dec_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_ACQB)
# 5374 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC_DEC_WB)
# 5377 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC_RB)
# 5381 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_DEC)
# 5385 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_FUNC__(dec)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5389 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC_FUNC__(dec_read_mb)(var);
#endif
# 5391 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_ddrb)(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC_FUNC__(dec)(var);
#else
# 5398 "../include/internal/ethr_atomics.h"
    ETHR_ATMC_FUNC__(dec_rb)(var);
#endif
# 5400 "../include/internal/ethr_atomics.h"
}


/* --- read_band() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_AND_RETOLD)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
# 5412 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
# 5414 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
# 5416 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
# 5418 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
# 5420 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5422 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5424 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5426 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5428 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5430 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5432 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#else
# 5434 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_band()!"
#endif
# 5436 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
# 5445 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
# 5448 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
# 5450 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
# 5453 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
# 5456 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5459 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5461 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5464 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5466 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5469 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5472 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 5475 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_band_rb()!"
#endif
# 5477 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
# 5486 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
# 5489 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
# 5491 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
# 5494 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
# 5497 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5500 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5502 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5505 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5507 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5510 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5513 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#else
# 5516 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_band_wb()!"
#endif
# 5518 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
# 5527 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
# 5530 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
# 5533 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
# 5535 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
# 5538 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5541 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5543 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5546 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5549 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5551 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5554 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5557 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_band_acqb()!"
#endif
# 5559 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
# 5568 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
# 5571 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
# 5574 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
# 5576 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
# 5579 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5582 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5584 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5587 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5590 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5592 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5595 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#else
# 5598 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_band_relb()!"
#endif
# 5600 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_AND_RETOLD_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RELB)
# 5609 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_ACQB)
# 5612 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_WB)
# 5615 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD_RB)
# 5619 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_AND_RETOLD)
# 5623 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5627 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5629 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5632 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5635 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5639 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5643 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5647 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_band_mb()!"
#endif
# 5649 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(read_band)(var, val);
#else
# 5657 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(read_band_rb)(var, val);
#endif
# 5659 "../include/internal/ethr_atomics.h"
}


/* --- read_bor() --- */


static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_OR_RETOLD)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
# 5671 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
# 5673 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
# 5675 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
# 5677 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
# 5679 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5681 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5683 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5685 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5687 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5689 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5691 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#else
# 5693 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_bor()!"
#endif
# 5695 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_rb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
# 5704 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
# 5707 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
# 5709 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
# 5712 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
# 5715 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5718 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5720 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5723 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5725 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5728 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5731 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 5734 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_bor_rb()!"
#endif
# 5736 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_wb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
# 5745 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
# 5748 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
# 5750 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
# 5753 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
# 5756 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5759 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5761 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5764 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5766 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5769 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5772 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
#else
# 5775 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_bor_wb()!"
#endif
# 5777 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_acqb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
# 5786 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
# 5789 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
# 5792 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
# 5794 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
# 5797 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5800 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5802 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5805 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5808 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5810 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5813 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5816 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_bor_acqb()!"
#endif
# 5818 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_relb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
# 5827 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
# 5830 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
# 5833 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
# 5835 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
# 5838 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5841 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5843 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5846 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5849 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5851 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5854 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#else
# 5857 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_bor_relb()!"
#endif
# 5859 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
#if defined(ETHR_HAVE_NATMC_OR_RETOLD_MB)
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_mb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RELB)
# 5868 "../include/internal/ethr_atomics.h"
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_relb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_ACQB)
# 5871 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_acqb)(var, (ETHR_NAINT_T__) val);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_WB)
# 5874 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_wb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD_RB)
# 5878 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold_rb)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_OR_RETOLD)
# 5882 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, (ETHR_NAINT_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_MB)
# 5886 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RELB)
# 5888 "../include/internal/ethr_atomics.h"
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_ACQB)
# 5891 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_WB)
# 5894 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG_RB)
# 5898 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC_CMPXCHG)
# 5902 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC_CMPXCHG_FALLBACK__(ETHR_NATMC_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 5906 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic_read_bor_mb()!"
#endif
# 5908 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC_FUNC__(read_bor)(var, val);
#else
# 5916 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC_FUNC__(read_bor_rb)(var, val);
#endif
# 5918 "../include/internal/ethr_atomics.h"
}

#endif /* ETHR_ATMC_INLINE__ */
# 5921 "../include/internal/ethr_atomics.h"


/* ---------- 32-bit atomic implementation ---------- */


#ifdef ETHR_NEED_ATMC32_PROTOTYPES__
ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var);
void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val);
void ethr_atomic32_inc(ethr_atomic32_t *var);
void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var);
void ethr_atomic32_inc_rb(ethr_atomic32_t *var);
void ethr_atomic32_inc_wb(ethr_atomic32_t *var);
void ethr_atomic32_inc_acqb(ethr_atomic32_t *var);
void ethr_atomic32_inc_relb(ethr_atomic32_t *var);
void ethr_atomic32_inc_mb(ethr_atomic32_t *var);
void ethr_atomic32_dec(ethr_atomic32_t *var);
void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var);
void ethr_atomic32_dec_rb(ethr_atomic32_t *var);
void ethr_atomic32_dec_wb(ethr_atomic32_t *var);
void ethr_atomic32_dec_acqb(ethr_atomic32_t *var);
void ethr_atomic32_dec_relb(ethr_atomic32_t *var);
void ethr_atomic32_dec_mb(ethr_atomic32_t *var);
ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val);
ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val);
#endif /* ETHR_NEED_ATMC32_PROTOTYPES__ */
# 6020 "../include/internal/ethr_atomics.h"

#if (defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) \
     && (defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__)))

#if !defined(ETHR_NATMC32_BITS__)
#  error "Missing native atomic implementation"
#elif ETHR_NATMC32_BITS__ == 64
# 6027 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
#    define ETHR_HAVE_NATMC32_CMPXCHG 1
#  endif
# 6031 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
#    define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
#  endif
# 6035 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
#    define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
#  endif
# 6039 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
#    define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
#  endif
# 6043 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
#    define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
#  endif
# 6047 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
#    define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
#  endif
# 6051 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
#    define ETHR_HAVE_NATMC32_XCHG 1
#  endif
# 6055 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
#    define ETHR_HAVE_NATMC32_XCHG_RB 1
#  endif
# 6059 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
#    define ETHR_HAVE_NATMC32_XCHG_WB 1
#  endif
# 6063 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
#    define ETHR_HAVE_NATMC32_XCHG_ACQB 1
#  endif
# 6067 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
#    define ETHR_HAVE_NATMC32_XCHG_RELB 1
#  endif
# 6071 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
#    define ETHR_HAVE_NATMC32_XCHG_MB 1
#  endif
# 6075 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
#    define ETHR_HAVE_NATMC32_SET 1
#  endif
# 6079 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
#    define ETHR_HAVE_NATMC32_SET_RB 1
#  endif
# 6083 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
#    define ETHR_HAVE_NATMC32_SET_WB 1
#  endif
# 6087 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
#    define ETHR_HAVE_NATMC32_SET_ACQB 1
#  endif
# 6091 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
#    define ETHR_HAVE_NATMC32_SET_RELB 1
#  endif
# 6095 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
#    define ETHR_HAVE_NATMC32_SET_MB 1
#  endif
# 6099 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
#    define ETHR_HAVE_NATMC32_INIT 1
#  endif
# 6103 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
#    define ETHR_HAVE_NATMC32_INIT_RB 1
#  endif
# 6107 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
#    define ETHR_HAVE_NATMC32_INIT_WB 1
#  endif
# 6111 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
#    define ETHR_HAVE_NATMC32_INIT_ACQB 1
#  endif
# 6115 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
#    define ETHR_HAVE_NATMC32_INIT_RELB 1
#  endif
# 6119 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
#    define ETHR_HAVE_NATMC32_INIT_MB 1
#  endif
# 6123 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
#    define ETHR_HAVE_NATMC32_ADD_RETURN 1
#  endif
# 6127 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
#  endif
# 6131 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
#  endif
# 6135 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
#  endif
# 6139 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
#  endif
# 6143 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
#  endif
# 6147 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
#    define ETHR_HAVE_NATMC32_READ 1
#  endif
# 6151 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
#    define ETHR_HAVE_NATMC32_READ_RB 1
#  endif
# 6155 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
#    define ETHR_HAVE_NATMC32_READ_WB 1
#  endif
# 6159 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
#    define ETHR_HAVE_NATMC32_READ_ACQB 1
#  endif
# 6163 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
#    define ETHR_HAVE_NATMC32_READ_RELB 1
#  endif
# 6167 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
#    define ETHR_HAVE_NATMC32_READ_MB 1
#  endif
# 6171 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
#    define ETHR_HAVE_NATMC32_INC_RETURN 1
#  endif
# 6175 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
#    define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
#  endif
# 6179 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
#    define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
#  endif
# 6183 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
#    define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
#  endif
# 6187 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
#    define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
#  endif
# 6191 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
#    define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
#  endif
# 6195 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
#    define ETHR_HAVE_NATMC32_DEC_RETURN 1
#  endif
# 6199 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
#  endif
# 6203 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
#  endif
# 6207 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
#  endif
# 6211 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
#  endif
# 6215 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
#  endif
# 6219 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
#    define ETHR_HAVE_NATMC32_ADD 1
#  endif
# 6223 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
#    define ETHR_HAVE_NATMC32_ADD_RB 1
#  endif
# 6227 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
#    define ETHR_HAVE_NATMC32_ADD_WB 1
#  endif
# 6231 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
#    define ETHR_HAVE_NATMC32_ADD_ACQB 1
#  endif
# 6235 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
#    define ETHR_HAVE_NATMC32_ADD_RELB 1
#  endif
# 6239 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
#    define ETHR_HAVE_NATMC32_ADD_MB 1
#  endif
# 6243 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
#    define ETHR_HAVE_NATMC32_INC 1
#  endif
# 6247 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
#    define ETHR_HAVE_NATMC32_INC_RB 1
#  endif
# 6251 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
#    define ETHR_HAVE_NATMC32_INC_WB 1
#  endif
# 6255 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
#    define ETHR_HAVE_NATMC32_INC_ACQB 1
#  endif
# 6259 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
#    define ETHR_HAVE_NATMC32_INC_RELB 1
#  endif
# 6263 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
#    define ETHR_HAVE_NATMC32_INC_MB 1
#  endif
# 6267 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
#    define ETHR_HAVE_NATMC32_DEC 1
#  endif
# 6271 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
#    define ETHR_HAVE_NATMC32_DEC_RB 1
#  endif
# 6275 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
#    define ETHR_HAVE_NATMC32_DEC_WB 1
#  endif
# 6279 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
#    define ETHR_HAVE_NATMC32_DEC_ACQB 1
#  endif
# 6283 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
#    define ETHR_HAVE_NATMC32_DEC_RELB 1
#  endif
# 6287 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
#    define ETHR_HAVE_NATMC32_DEC_MB 1
#  endif
# 6291 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
#    define ETHR_HAVE_NATMC32_AND_RETOLD 1
#  endif
# 6295 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
#  endif
# 6299 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
#  endif
# 6303 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
#  endif
# 6307 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
#  endif
# 6311 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
#  endif
# 6315 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
#    define ETHR_HAVE_NATMC32_OR_RETOLD 1
#  endif
# 6319 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
#  endif
# 6323 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
#  endif
# 6327 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
#  endif
# 6331 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
#  endif
# 6335 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
#  endif
# 6339 "../include/internal/ethr_atomics.h"
#elif ETHR_NATMC32_BITS__ == 32
# 6340 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
#    define ETHR_HAVE_NATMC32_CMPXCHG 1
#  endif
# 6344 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
#    define ETHR_HAVE_NATMC32_CMPXCHG_RB 1
#  endif
# 6348 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
#    define ETHR_HAVE_NATMC32_CMPXCHG_WB 1
#  endif
# 6352 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
#    define ETHR_HAVE_NATMC32_CMPXCHG_ACQB 1
#  endif
# 6356 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
#    define ETHR_HAVE_NATMC32_CMPXCHG_RELB 1
#  endif
# 6360 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_CMPXCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
#    define ETHR_HAVE_NATMC32_CMPXCHG_MB 1
#  endif
# 6364 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
#    define ETHR_HAVE_NATMC32_XCHG 1
#  endif
# 6368 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
#    define ETHR_HAVE_NATMC32_XCHG_RB 1
#  endif
# 6372 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
#    define ETHR_HAVE_NATMC32_XCHG_WB 1
#  endif
# 6376 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
#    define ETHR_HAVE_NATMC32_XCHG_ACQB 1
#  endif
# 6380 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
#    define ETHR_HAVE_NATMC32_XCHG_RELB 1
#  endif
# 6384 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_XCHG_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
#    define ETHR_HAVE_NATMC32_XCHG_MB 1
#  endif
# 6388 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
#    define ETHR_HAVE_NATMC32_SET 1
#  endif
# 6392 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
#    define ETHR_HAVE_NATMC32_SET_RB 1
#  endif
# 6396 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
#    define ETHR_HAVE_NATMC32_SET_WB 1
#  endif
# 6400 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
#    define ETHR_HAVE_NATMC32_SET_ACQB 1
#  endif
# 6404 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
#    define ETHR_HAVE_NATMC32_SET_RELB 1
#  endif
# 6408 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_SET_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
#    define ETHR_HAVE_NATMC32_SET_MB 1
#  endif
# 6412 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
#    define ETHR_HAVE_NATMC32_INIT 1
#  endif
# 6416 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
#    define ETHR_HAVE_NATMC32_INIT_RB 1
#  endif
# 6420 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
#    define ETHR_HAVE_NATMC32_INIT_WB 1
#  endif
# 6424 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
#    define ETHR_HAVE_NATMC32_INIT_ACQB 1
#  endif
# 6428 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
#    define ETHR_HAVE_NATMC32_INIT_RELB 1
#  endif
# 6432 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INIT_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
#    define ETHR_HAVE_NATMC32_INIT_MB 1
#  endif
# 6436 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
#    define ETHR_HAVE_NATMC32_ADD_RETURN 1
#  endif
# 6440 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_RB 1
#  endif
# 6444 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_WB 1
#  endif
# 6448 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_ACQB 1
#  endif
# 6452 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_RELB 1
#  endif
# 6456 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
#    define ETHR_HAVE_NATMC32_ADD_RETURN_MB 1
#  endif
# 6460 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
#    define ETHR_HAVE_NATMC32_READ 1
#  endif
# 6464 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
#    define ETHR_HAVE_NATMC32_READ_RB 1
#  endif
# 6468 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
#    define ETHR_HAVE_NATMC32_READ_WB 1
#  endif
# 6472 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
#    define ETHR_HAVE_NATMC32_READ_ACQB 1
#  endif
# 6476 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
#    define ETHR_HAVE_NATMC32_READ_RELB 1
#  endif
# 6480 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_READ_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
#    define ETHR_HAVE_NATMC32_READ_MB 1
#  endif
# 6484 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
#    define ETHR_HAVE_NATMC32_INC_RETURN 1
#  endif
# 6488 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
#    define ETHR_HAVE_NATMC32_INC_RETURN_RB 1
#  endif
# 6492 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
#    define ETHR_HAVE_NATMC32_INC_RETURN_WB 1
#  endif
# 6496 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
#    define ETHR_HAVE_NATMC32_INC_RETURN_ACQB 1
#  endif
# 6500 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
#    define ETHR_HAVE_NATMC32_INC_RETURN_RELB 1
#  endif
# 6504 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
#    define ETHR_HAVE_NATMC32_INC_RETURN_MB 1
#  endif
# 6508 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
#    define ETHR_HAVE_NATMC32_DEC_RETURN 1
#  endif
# 6512 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_RB 1
#  endif
# 6516 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_WB 1
#  endif
# 6520 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_ACQB 1
#  endif
# 6524 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_RELB 1
#  endif
# 6528 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RETURN_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
#    define ETHR_HAVE_NATMC32_DEC_RETURN_MB 1
#  endif
# 6532 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
#    define ETHR_HAVE_NATMC32_ADD 1
#  endif
# 6536 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
#    define ETHR_HAVE_NATMC32_ADD_RB 1
#  endif
# 6540 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
#    define ETHR_HAVE_NATMC32_ADD_WB 1
#  endif
# 6544 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
#    define ETHR_HAVE_NATMC32_ADD_ACQB 1
#  endif
# 6548 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
#    define ETHR_HAVE_NATMC32_ADD_RELB 1
#  endif
# 6552 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_ADD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
#    define ETHR_HAVE_NATMC32_ADD_MB 1
#  endif
# 6556 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
#    define ETHR_HAVE_NATMC32_INC 1
#  endif
# 6560 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
#    define ETHR_HAVE_NATMC32_INC_RB 1
#  endif
# 6564 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
#    define ETHR_HAVE_NATMC32_INC_WB 1
#  endif
# 6568 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
#    define ETHR_HAVE_NATMC32_INC_ACQB 1
#  endif
# 6572 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
#    define ETHR_HAVE_NATMC32_INC_RELB 1
#  endif
# 6576 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_INC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
#    define ETHR_HAVE_NATMC32_INC_MB 1
#  endif
# 6580 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
#    define ETHR_HAVE_NATMC32_DEC 1
#  endif
# 6584 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
#    define ETHR_HAVE_NATMC32_DEC_RB 1
#  endif
# 6588 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
#    define ETHR_HAVE_NATMC32_DEC_WB 1
#  endif
# 6592 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
#    define ETHR_HAVE_NATMC32_DEC_ACQB 1
#  endif
# 6596 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
#    define ETHR_HAVE_NATMC32_DEC_RELB 1
#  endif
# 6600 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_DEC_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
#    define ETHR_HAVE_NATMC32_DEC_MB 1
#  endif
# 6604 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
#    define ETHR_HAVE_NATMC32_AND_RETOLD 1
#  endif
# 6608 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_RB 1
#  endif
# 6612 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_WB 1
#  endif
# 6616 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_ACQB 1
#  endif
# 6620 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_RELB 1
#  endif
# 6624 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_AND_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
#    define ETHR_HAVE_NATMC32_AND_RETOLD_MB 1
#  endif
# 6628 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
#    define ETHR_HAVE_NATMC32_OR_RETOLD 1
#  endif
# 6632 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_RB 1
#  endif
# 6636 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_WB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_WB 1
#  endif
# 6640 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_ACQB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_ACQB 1
#  endif
# 6644 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_RELB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_RELB 1
#  endif
# 6648 "../include/internal/ethr_atomics.h"
#  undef ETHR_HAVE_NATMC32_OR_RETOLD_MB
#  ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
#    define ETHR_HAVE_NATMC32_OR_RETOLD_MB 1
#  endif
# 6652 "../include/internal/ethr_atomics.h"
#else
# 6653 "../include/internal/ethr_atomics.h"
#  error "Invalid native atomic size"
#endif
# 6655 "../include/internal/ethr_atomics.h"

#if (!defined(ETHR_HAVE_NATMC32_CMPXCHG) \
     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RB) \
     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_WB) \
     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB) \
     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB) \
     && !defined(ETHR_HAVE_NATMC32_CMPXCHG_MB))
#  error "No native cmpxchg() op available"
#endif
# 6664 "../include/internal/ethr_atomics.h"


/*
 * Read op used together with cmpxchg() fallback when no native op present.
 */
#if defined(ETHR_HAVE_NATMC32_READ)
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC32_FUNC__(read)(VAR)
#elif defined(ETHR_HAVE_NATMC32_READ_RB)
# 6673 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC32_FUNC__(read_rb)(VAR)
#elif defined(ETHR_HAVE_NATMC32_READ_WB)
# 6676 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC32_FUNC__(read_wb)(VAR)
#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
# 6679 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC32_FUNC__(read_acqb)(VAR)
#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
# 6682 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC32_FUNC__(read_relb)(VAR)
#elif defined(ETHR_HAVE_NATMC32_READ_MB)
# 6685 "../include/internal/ethr_atomics.h"
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ETHR_NATMC32_FUNC__(read_mb)(VAR)
#else
# 6688 "../include/internal/ethr_atomics.h"
/*
 * We have no native read() op; guess zero and then use the
 * the atomics actual value returned from cmpxchg().
 */
#define ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR) \
  ((ETHR_NAINT32_T__) 0)
#endif
# 6695 "../include/internal/ethr_atomics.h"

/*
 * Native cmpxchg() fallback used when no native op present.
 */
#define ETHR_NATMC32_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \
do { \
    ethr_sint32_t AVAL; \
    ETHR_NAINT32_T__ new__, act__, exp__; \
    act__ = ETHR_NATMC32_CMPXCHG_FALLBACK_READ__(VAR); \
    do { \
	exp__ = act__; \
	AVAL = (ethr_sint32_t) act__; \
	{ OPS; } \
	new__ = (ETHR_NAINT32_T__) AVAL; \
	act__ = CMPXCHG(VAR, new__, exp__); \
    } while (__builtin_expect(act__ != exp__, 0)); \
} while (0)



/* --- addr() --- */

static ETHR_INLINE ethr_sint32_t *ETHR_ATMC32_FUNC__(addr)(ethr_atomic32_t *var)
{
    return (ethr_sint32_t *) ETHR_NATMC32_ADDR_FUNC__(var);

}


/* --- cmpxchg() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_CMPXCHG)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6733 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6735 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6737 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6739 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6741 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#else
# 6743 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_cmpxchg()!"
#endif
# 6745 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6754 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6757 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6759 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6762 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6765 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 6768 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_cmpxchg_rb()!"
#endif
# 6770 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6779 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6782 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6784 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6787 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6790 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#else
# 6793 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_cmpxchg_wb()!"
#endif
# 6795 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6804 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6807 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6810 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6812 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6815 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 6818 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_cmpxchg_acqb()!"
#endif
# 6820 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6829 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6832 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6835 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6837 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6840 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#else
# 6843 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_cmpxchg_relb()!"
#endif
# 6845 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_mb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6854 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6857 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6860 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_wb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6864 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg_rb)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6868 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, (ETHR_NAINT32_T__) val, (ETHR_NAINT32_T__) old_val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 6872 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_cmpxchg_mb()!"
#endif
# 6874 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(cmpxchg)(var, val, old_val);
#else
# 6882 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, val, old_val);
#endif
# 6884 "../include/internal/ethr_atomics.h"
}


/* --- xchg() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_XCHG)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
# 6896 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
# 6898 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
# 6900 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
# 6902 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
# 6904 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6906 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6908 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6910 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6912 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6914 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6916 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#else
# 6918 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_xchg()!"
#endif
# 6920 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_XCHG_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG)
# 6929 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
# 6932 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
# 6934 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
# 6937 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
# 6940 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6943 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6945 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6948 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6950 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6953 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6956 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 6959 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_xchg_rb()!"
#endif
# 6961 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_XCHG_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG)
# 6970 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
# 6973 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
# 6975 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
# 6978 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
# 6981 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 6984 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 6986 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 6989 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 6991 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 6994 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 6997 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
#else
# 7000 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_xchg_wb()!"
#endif
# 7002 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
# 7011 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_XCHG)
# 7014 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
# 7017 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
# 7019 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
# 7022 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7025 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7027 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7030 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7033 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7035 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7038 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7041 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_xchg_acqb()!"
#endif
# 7043 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_XCHG_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
# 7052 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG)
# 7055 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_MB)
# 7058 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
# 7060 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
# 7063 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7066 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7068 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7071 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7074 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7076 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7079 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#else
# 7082 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_xchg_relb()!"
#endif
# 7084 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_XCHG_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RELB)
# 7093 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_XCHG_ACQB)
# 7096 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_XCHG_WB)
# 7099 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_XCHG_RB)
# 7103 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_XCHG)
# 7107 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7111 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7113 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7116 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval = val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7119 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7123 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7127 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7131 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_xchg_mb()!"
#endif
# 7133 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(xchg)(var, val);
#else
# 7141 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
#endif
# 7143 "../include/internal/ethr_atomics.h"
}


/* --- set() --- */


static ETHR_INLINE void ETHR_ATMC32_FUNC__(set)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_SET)
    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RB)
# 7154 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_WB)
# 7156 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
# 7158 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
# 7160 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_MB)
# 7162 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
#else
# 7164 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(xchg)(var, val);
#endif
# 7166 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_SET_RB)
    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET)
# 7173 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_SET_MB)
# 7176 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_WB)
# 7178 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
# 7181 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
# 7184 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 7187 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
#endif
# 7189 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_SET_WB)
    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET)
# 7196 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_MB)
# 7199 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RB)
# 7201 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
# 7204 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
# 7207 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
#else
# 7210 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(xchg_wb)(var, val);
#endif
# 7212 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_SET_ACQB)
    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RB)
# 7219 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_SET)
# 7222 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_SET_MB)
# 7225 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_WB)
# 7227 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
# 7230 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7233 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(xchg_acqb)(var, val);
#endif
# 7235 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_SET_RELB)
    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_WB)
# 7242 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET)
# 7245 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_MB)
# 7248 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RB)
# 7250 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
# 7253 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
#else
# 7256 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(xchg_relb)(var, val);
#endif
# 7258 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_SET_MB)
    ETHR_NATMC32_FUNC__(set_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_RELB)
# 7265 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_SET_ACQB)
# 7268 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_SET_WB)
# 7271 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore);
    ETHR_NATMC32_FUNC__(set_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_SET_RB)
# 7275 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_SET)
# 7279 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7283 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(xchg_mb)(var, val);
#endif
# 7285 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC32_FUNC__(set)(var, val);
#else
# 7292 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set_rb)(var, val);
#endif
# 7294 "../include/internal/ethr_atomics.h"
}


/* --- init() --- */


static ETHR_INLINE void ETHR_ATMC32_FUNC__(init)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_INIT)
    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
# 7305 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
# 7307 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
# 7309 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
# 7311 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
# 7313 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#else
# 7315 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set)(var, val);
#endif
# 7317 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_INIT_RB)
    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT)
# 7324 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
# 7327 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
# 7329 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
# 7332 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
# 7335 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 7338 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set_rb)(var, val);
#endif
# 7340 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_INIT_WB)
    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT)
# 7347 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
# 7350 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
# 7352 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
# 7355 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
# 7358 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
#else
# 7361 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set_wb)(var, val);
#endif
# 7363 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_INIT_ACQB)
    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
# 7370 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INIT)
# 7373 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
# 7376 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
# 7378 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
# 7381 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7384 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set_acqb)(var, val);
#endif
# 7386 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_INIT_RELB)
    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
# 7393 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT)
# 7396 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_MB)
# 7399 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
# 7401 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
# 7404 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
#else
# 7407 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set_relb)(var, val);
#endif
# 7409 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_INIT_MB)
    ETHR_NATMC32_FUNC__(init_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_RELB)
# 7416 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(init_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INIT_ACQB)
# 7419 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_INIT_WB)
# 7422 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(init_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INIT_RB)
# 7426 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INIT)
# 7430 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7434 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(set_mb)(var, val);
#endif
# 7436 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC32_FUNC__(init)(var, val);
#else
# 7443 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(init_rb)(var, val);
#endif
# 7445 "../include/internal/ethr_atomics.h"
}


/* --- add_read() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_ADD_RETURN)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
# 7457 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
# 7459 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
# 7461 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
# 7463 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
# 7465 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7467 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7469 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7471 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7473 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7475 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7477 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#else
# 7479 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_add_read()!"
#endif
# 7481 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
# 7490 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
# 7493 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
# 7495 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
# 7498 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
# 7501 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7504 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7506 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7509 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7511 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7514 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7517 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 7520 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_add_read_rb()!"
#endif
# 7522 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
# 7531 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
# 7534 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
# 7536 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
# 7539 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
# 7542 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7545 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7547 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7550 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7552 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7555 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7558 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#else
# 7561 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_add_read_wb()!"
#endif
# 7563 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
# 7572 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
# 7575 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
# 7578 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
# 7580 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
# 7583 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7586 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7588 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7591 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7594 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7596 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7599 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7602 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_add_read_acqb()!"
#endif
# 7604 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
# 7613 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
# 7616 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
# 7619 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
# 7621 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
# 7624 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7627 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7629 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7632 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7635 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7637 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7640 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#else
# 7643 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_add_read_relb()!"
#endif
# 7645 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_ADD_RETURN_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RELB)
# 7654 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_ACQB)
# 7657 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_WB)
# 7660 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN_RB)
# 7664 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RETURN)
# 7668 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 7672 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 7674 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 7677 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, aval += val; res = aval);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 7680 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 7684 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 7688 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, aval += val; res = aval);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7692 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_add_read_mb()!"
#endif
# 7694 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(add_read)(var, val);
#else
# 7702 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
#endif
# 7704 "../include/internal/ethr_atomics.h"
}


/* --- read() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_READ)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RB)
# 7716 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_WB)
# 7718 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
# 7720 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
# 7722 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_MB)
# 7724 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
#else
# 7726 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(cmpxchg)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
# 7728 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_rb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_READ_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ)
# 7737 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_READ_MB)
# 7740 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_WB)
# 7742 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
# 7745 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
# 7748 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 7751 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
# 7753 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_wb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_READ_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ)
# 7762 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_MB)
# 7765 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RB)
# 7767 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
# 7770 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
# 7773 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
#else
# 7776 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(cmpxchg_wb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
# 7778 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_acqb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_READ_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RB)
# 7787 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC32_READ)
# 7790 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC32_READ_MB)
# 7793 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_WB)
# 7795 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
# 7798 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
# 7801 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(cmpxchg_acqb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
# 7803 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_relb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_READ_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_WB)
# 7812 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ)
# 7815 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_MB)
# 7818 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RB)
# 7820 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
# 7823 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
#else
# 7826 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(cmpxchg_relb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
# 7828 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_READ_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_RELB)
# 7837 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC32_READ_ACQB)
# 7840 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_READ_WB)
# 7843 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC32_READ_RB)
# 7847 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore);
#elif defined(ETHR_HAVE_NATMC32_READ)
# 7851 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#else
# 7855 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(cmpxchg_mb)(var, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__, (ethr_sint32_t) ETHR_UNUSUAL_SINT32_VAL__);
#endif
# 7857 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_ddrb)(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(read)(var);
#else
# 7865 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(read_rb)(var);
#endif
# 7867 "../include/internal/ethr_atomics.h"
}


/* --- inc_read() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_INC_RETURN)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
# 7879 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
# 7881 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
# 7883 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
# 7885 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
# 7887 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
#else
# 7889 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) 1);
#endif
# 7891 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_rb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
# 7900 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
# 7903 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
# 7905 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
# 7908 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
# 7911 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 7914 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) 1);
#endif
# 7916 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_wb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
# 7925 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
# 7928 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
# 7930 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
# 7933 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
# 7936 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
#else
# 7939 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) 1);
#endif
# 7941 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_acqb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
# 7950 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
# 7953 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
# 7956 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
# 7958 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
# 7961 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 7964 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) 1);
#endif
# 7966 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_relb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
# 7975 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
# 7978 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
# 7981 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
# 7983 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
# 7986 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
#else
# 7989 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) 1);
#endif
# 7991 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_INC_RETURN_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RELB)
# 8000 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_ACQB)
# 8003 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_WB)
# 8006 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN_RB)
# 8010 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RETURN)
# 8014 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8018 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) 1);
#endif
# 8020 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_ddrb)(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(inc_read)(var);
#else
# 8028 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(inc_read_rb)(var);
#endif
# 8030 "../include/internal/ethr_atomics.h"
}


/* --- dec_read() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_DEC_RETURN)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
# 8042 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
# 8044 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
# 8046 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
# 8048 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
# 8050 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#else
# 8052 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read)(var, (ethr_sint32_t) -1);
#endif
# 8054 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_rb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
# 8063 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
# 8066 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
# 8068 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
# 8071 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
# 8074 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 8077 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_rb)(var, (ethr_sint32_t) -1);
#endif
# 8079 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_wb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
# 8088 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
# 8091 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
# 8093 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
# 8096 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
# 8099 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
#else
# 8102 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_wb)(var, (ethr_sint32_t) -1);
#endif
# 8104 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_acqb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
# 8113 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
# 8116 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
# 8119 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
# 8121 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
# 8124 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8127 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_acqb)(var, (ethr_sint32_t) -1);
#endif
# 8129 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_relb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
# 8138 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
# 8141 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
# 8144 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
# 8146 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
# 8149 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
#else
# 8152 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_relb)(var, (ethr_sint32_t) -1);
#endif
# 8154 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_DEC_RETURN_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RELB)
# 8163 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_ACQB)
# 8166 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_WB)
# 8169 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN_RB)
# 8173 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RETURN)
# 8177 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8181 "../include/internal/ethr_atomics.h"
    res = ETHR_ATMC32_FUNC__(add_read_mb)(var, (ethr_sint32_t) -1);
#endif
# 8183 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_ddrb)(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(dec_read)(var);
#else
# 8191 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(dec_read_rb)(var);
#endif
# 8193 "../include/internal/ethr_atomics.h"
}


/* --- add() --- */


static ETHR_INLINE void ETHR_ATMC32_FUNC__(add)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_ADD)
    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
# 8204 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
# 8206 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
# 8208 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
# 8210 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
# 8212 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
#else
# 8214 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(add_read)(var, val);
#endif
# 8216 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_ADD_RB)
    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD)
# 8223 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
# 8226 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
# 8228 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
# 8231 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
# 8234 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 8237 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
#endif
# 8239 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_ADD_WB)
    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD)
# 8246 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
# 8249 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
# 8251 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
# 8254 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
# 8257 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
#else
# 8260 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(add_read_wb)(var, val);
#endif
# 8262 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_ADD_ACQB)
    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
# 8269 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD)
# 8272 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
# 8275 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
# 8277 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
# 8280 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8283 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(add_read_acqb)(var, val);
#endif
# 8285 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_ADD_RELB)
    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
# 8292 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD)
# 8295 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_MB)
# 8298 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
# 8300 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
# 8303 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
#else
# 8306 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(add_read_relb)(var, val);
#endif
# 8308 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#if defined(ETHR_HAVE_NATMC32_ADD_MB)
    ETHR_NATMC32_FUNC__(add_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_RELB)
# 8315 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(add_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_ACQB)
# 8318 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_ADD_WB)
# 8321 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD_RB)
# 8325 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_ADD)
# 8329 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8333 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(add_read_mb)(var, val);
#endif
# 8335 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC32_FUNC__(add)(var, val);
#else
# 8342 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(add_rb)(var, val);
#endif
# 8344 "../include/internal/ethr_atomics.h"
}


/* --- inc() --- */


static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_INC)
    ETHR_NATMC32_FUNC__(inc)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RB)
# 8355 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_WB)
# 8357 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
# 8359 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
# 8361 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_MB)
# 8363 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_mb)(var);
#else
# 8365 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(inc_read)(var);
#endif
# 8367 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_rb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_INC_RB)
    ETHR_NATMC32_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC)
# 8374 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INC_MB)
# 8377 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_WB)
# 8379 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
# 8382 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
# 8385 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 8388 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(inc_read_rb)(var);
#endif
# 8390 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_wb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_INC_WB)
    ETHR_NATMC32_FUNC__(inc_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC)
# 8397 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(inc)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_MB)
# 8400 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RB)
# 8402 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
# 8405 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
# 8408 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(inc_relb)(var);
#else
# 8411 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(inc_read_wb)(var);
#endif
# 8413 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_acqb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_INC_ACQB)
    ETHR_NATMC32_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RB)
# 8420 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC)
# 8423 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_MB)
# 8426 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_WB)
# 8428 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
# 8431 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8434 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(inc_read_acqb)(var);
#endif
# 8436 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_relb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_INC_RELB)
    ETHR_NATMC32_FUNC__(inc_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_WB)
# 8443 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC)
# 8446 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_MB)
# 8449 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RB)
# 8451 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
# 8454 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc_acqb)(var);
#else
# 8457 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(inc_read_relb)(var);
#endif
# 8459 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_INC_MB)
    ETHR_NATMC32_FUNC__(inc_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_RELB)
# 8466 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(inc_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_ACQB)
# 8469 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_INC_WB)
# 8472 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC_RB)
# 8476 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_INC)
# 8480 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(inc)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8484 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(inc_read_mb)(var);
#endif
# 8486 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_ddrb)(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC32_FUNC__(inc)(var);
#else
# 8493 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(inc_rb)(var);
#endif
# 8495 "../include/internal/ethr_atomics.h"
}


/* --- dec() --- */


static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_DEC)
    ETHR_NATMC32_FUNC__(dec)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
# 8506 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
# 8508 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
# 8510 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
# 8512 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
# 8514 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_mb)(var);
#else
# 8516 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(dec_read)(var);
#endif
# 8518 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_rb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_DEC_RB)
    ETHR_NATMC32_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC)
# 8525 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
# 8528 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
# 8530 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
# 8533 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_acqb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
# 8536 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 8539 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(dec_read_rb)(var);
#endif
# 8541 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_wb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_DEC_WB)
    ETHR_NATMC32_FUNC__(dec_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC)
# 8548 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(dec)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
# 8551 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
# 8553 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
# 8556 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
# 8559 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_FUNC__(dec_relb)(var);
#else
# 8562 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(dec_read_wb)(var);
#endif
# 8564 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_acqb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_DEC_ACQB)
    ETHR_NATMC32_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
# 8571 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC)
# 8574 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
# 8577 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
# 8579 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
# 8582 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8585 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(dec_read_acqb)(var);
#endif
# 8587 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_relb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_DEC_RELB)
    ETHR_NATMC32_FUNC__(dec_relb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
# 8594 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec_wb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC)
# 8597 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_MB)
# 8600 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
# 8602 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec_rb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
# 8605 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec_acqb)(var);
#else
# 8608 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(dec_read_relb)(var);
#endif
# 8610 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var)
{
#if defined(ETHR_HAVE_NATMC32_DEC_MB)
    ETHR_NATMC32_FUNC__(dec_mb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_RELB)
# 8617 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_FUNC__(dec_relb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_ACQB)
# 8620 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec_acqb)(var);
#elif defined(ETHR_HAVE_NATMC32_DEC_WB)
# 8623 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec_wb)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC_RB)
# 8627 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec_rb)(var);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_DEC)
# 8631 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_FUNC__(dec)(var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8635 "../include/internal/ethr_atomics.h"
    (void) ETHR_ATMC32_FUNC__(dec_read_mb)(var);
#endif
# 8637 "../include/internal/ethr_atomics.h"
}

static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_ddrb)(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ETHR_ATMC32_FUNC__(dec)(var);
#else
# 8644 "../include/internal/ethr_atomics.h"
    ETHR_ATMC32_FUNC__(dec_rb)(var);
#endif
# 8646 "../include/internal/ethr_atomics.h"
}


/* --- read_band() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_AND_RETOLD)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
# 8658 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
# 8660 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
# 8662 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
# 8664 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
# 8666 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8668 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8670 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8672 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8674 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8676 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8678 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#else
# 8680 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_band()!"
#endif
# 8682 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
# 8691 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
# 8694 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
# 8696 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
# 8699 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
# 8702 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8705 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8707 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8710 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8712 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8715 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8718 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 8721 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_band_rb()!"
#endif
# 8723 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
# 8732 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
# 8735 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
# 8737 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
# 8740 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
# 8743 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8746 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8748 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8751 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8753 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8756 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8759 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#else
# 8762 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_band_wb()!"
#endif
# 8764 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
# 8773 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
# 8776 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
# 8779 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
# 8781 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
# 8784 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8787 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8789 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8792 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8795 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8797 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8800 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8803 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_band_acqb()!"
#endif
# 8805 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
# 8814 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
# 8817 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
# 8820 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
# 8822 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
# 8825 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8828 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8830 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8833 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8836 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8838 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8841 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#else
# 8844 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_band_relb()!"
#endif
# 8846 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_AND_RETOLD_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RELB)
# 8855 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_ACQB)
# 8858 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_WB)
# 8861 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD_RB)
# 8865 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_AND_RETOLD)
# 8869 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8873 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8875 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8878 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval &= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8881 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8885 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8889 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 8893 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_band_mb()!"
#endif
# 8895 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(read_band)(var, val);
#else
# 8903 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(read_band_rb)(var, val);
#endif
# 8905 "../include/internal/ethr_atomics.h"
}


/* --- read_bor() --- */


static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_OR_RETOLD)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
# 8917 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
# 8919 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
# 8921 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
# 8923 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
# 8925 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8927 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8929 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8931 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8933 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8935 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8937 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#else
# 8939 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_bor()!"
#endif
# 8941 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_rb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
# 8950 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
# 8953 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
# 8955 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
# 8958 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
# 8961 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 8964 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 8966 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 8969 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 8971 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 8974 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 8977 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
# 8980 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_bor_rb()!"
#endif
# 8982 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_wb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
# 8991 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
# 8994 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
# 8996 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
# 8999 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
# 9002 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 9005 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 9007 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 9010 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 9012 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 9015 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 9018 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
#else
# 9021 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_bor_wb()!"
#endif
# 9023 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_acqb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
# 9032 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
# 9035 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
# 9038 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
# 9040 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
# 9043 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 9046 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 9048 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 9051 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 9054 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 9056 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 9059 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 9062 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_bor_acqb()!"
#endif
# 9064 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_relb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
# 9073 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
# 9076 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
# 9079 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
# 9081 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
# 9084 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 9087 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 9089 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 9092 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 9095 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 9097 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 9100 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#else
# 9103 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_bor_relb()!"
#endif
# 9105 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
#if defined(ETHR_HAVE_NATMC32_OR_RETOLD_MB)
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_mb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RELB)
# 9114 "../include/internal/ethr_atomics.h"
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_relb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_ACQB)
# 9117 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_acqb)(var, (ETHR_NAINT32_T__) val);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_WB)
# 9120 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_wb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD_RB)
# 9124 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold_rb)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_OR_RETOLD)
# 9128 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, (ETHR_NAINT32_T__) val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_MB)
# 9132 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_mb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RELB)
# 9134 "../include/internal/ethr_atomics.h"
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_relb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_ACQB)
# 9137 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_acqb), var, aval, res = aval; aval |= val);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_WB)
# 9140 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_wb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG_RB)
# 9144 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg_rb), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#elif defined(ETHR_HAVE_NATMC32_CMPXCHG)
# 9148 "../include/internal/ethr_atomics.h"
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_NATMC32_CMPXCHG_FALLBACK__(ETHR_NATMC32_FUNC__(cmpxchg), var, aval, res = aval; aval |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
# 9152 "../include/internal/ethr_atomics.h"
#error "Missing implementation of ethr_atomic32_read_bor_mb()!"
#endif
# 9154 "../include/internal/ethr_atomics.h"
    return res;
}

static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ETHR_ATMC32_FUNC__(read_bor)(var, val);
#else
# 9162 "../include/internal/ethr_atomics.h"
    return ETHR_ATMC32_FUNC__(read_bor_rb)(var, val);
#endif
# 9164 "../include/internal/ethr_atomics.h"
}

#endif /* ETHR_ATMC32_INLINE__ */
# 9167 "../include/internal/ethr_atomics.h"

#endif /* ETHR_ATOMICS_H__ */
# 9169 "../include/internal/ethr_atomics.h"
# 385 "../include/internal/ethread.h" 2

#if defined (ETHR_OSE_THREADS)
static ETHR_INLINE void
ose_yield(void)
{
    if (get_ptype(current_process()) == OS_PRI_PROC) {
        set_pri(get_pri(current_process()));
    } else {
        delay(1);
    }
}
#endif
# 397 "../include/internal/ethread.h"

#if defined(__GNUC__) && !defined(ETHR_OSE_THREADS)
#  ifndef ETHR_SPIN_BODY
#    if defined(__i386__) || defined(__x86_64__)
#      define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory")
#    elif defined(__ia64__)
# 403 "../include/internal/ethread.h"
#      define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory")
#    elif defined(__sparc__)
# 405 "../include/internal/ethread.h"
#      define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad")
#    else
# 407 "../include/internal/ethread.h"
#      define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
#    endif
# 409 "../include/internal/ethread.h"
#  endif
# 410 "../include/internal/ethread.h"
#elif defined(ETHR_WIN32_THREADS)
# 411 "../include/internal/ethread.h"
#  ifndef ETHR_SPIN_BODY
#    define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0)
#  endif
# 414 "../include/internal/ethread.h"
#elif defined(ETHR_OSE_THREADS)
# 415 "../include/internal/ethread.h"
#  ifndef ETHR_SPIN_BODY
#    define ETHR_SPIN_BODY ose_yield()
#  else
# 418 "../include/internal/ethread.h"
#    error "OSE should use ose_yield()"
#  endif
# 420 "../include/internal/ethread.h"
#endif
# 421 "../include/internal/ethread.h"

#ifndef ETHR_OSE_THREADS
#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
#else
# 425 "../include/internal/ethread.h"
#define ETHR_YIELD_AFTER_BUSY_LOOPS 0
#endif
# 427 "../include/internal/ethread.h"


#ifndef ETHR_SPIN_BODY
#  define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER
#endif
# 432 "../include/internal/ethread.h"

#ifndef ETHR_YIELD
#  if defined(ETHR_HAVE_SCHED_YIELD)
#    ifdef ETHR_HAVE_SCHED_H
#if 0 /* expanded by -frewrite-includes */
#      include <sched.h>
#endif /* expanded by -frewrite-includes */
# 436 "../include/internal/ethread.h"
# 437 "../include/internal/ethread.h"
#    endif
# 438 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#    include <errno.h>
#endif /* expanded by -frewrite-includes */
# 438 "../include/internal/ethread.h"
# 439 "../include/internal/ethread.h"
#    if defined(ETHR_SCHED_YIELD_RET_INT)
#      define ETHR_YIELD() (sched_yield() < 0 ? errno : 0)
#    else
# 442 "../include/internal/ethread.h"
#      define ETHR_YIELD() (sched_yield(), 0)
#    endif
# 444 "../include/internal/ethread.h"
#  elif defined(ETHR_HAVE_PTHREAD_YIELD)
# 445 "../include/internal/ethread.h"
#    if defined(ETHR_PTHREAD_YIELD_RET_INT)
#      define ETHR_YIELD() pthread_yield()
#    else
# 448 "../include/internal/ethread.h"
#      define ETHR_YIELD() (pthread_yield(), 0)
#    endif
# 450 "../include/internal/ethread.h"
#  elif defined(ETHR_OSE_THREADS)
# 451 "../include/internal/ethread.h"
#    define ETHR_YIELD() (ose_yield(), 0)
#  else
# 453 "../include/internal/ethread.h"
#    define ETHR_YIELD() (ethr_compiler_barrier(), 0)
#  endif
# 455 "../include/internal/ethread.h"
#endif
# 456 "../include/internal/ethread.h"

#if defined(VALGRIND) || defined(ETHR_OSE_THREADS)
/* mutex as fallback for spinlock for VALGRIND and OSE.
   OSE cannot use spinlocks as processes working on the
   same execution unit have a tendency to deadlock.
 */
#  undef ETHR_HAVE_NATIVE_SPINLOCKS
#  undef ETHR_HAVE_NATIVE_RWSPINLOCKS
#else
# 465 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#  include "ethr_optimized_fallbacks.h"
#endif /* expanded by -frewrite-includes */
# 465 "../include/internal/ethread.h"
# 1 "../include/internal/ethr_optimized_fallbacks.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Optimized fallbacks used when native ops are missing
 * Author: Rickard Green
 */

#ifndef ETHR_OPTIMIZED_FALLBACKS_H__
#define ETHR_OPTIMIZED_FALLBACKS_H__

#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

static ETHR_INLINE int
ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
{
    return 0;
}

#endif
# 40 "../include/internal/ethr_optimized_fallbacks.h"

#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK)
# 42 "../include/internal/ethr_optimized_fallbacks.h"
/* --- Native spinlocks using pthread spinlocks -------------------------- */
#define ETHR_HAVE_NATIVE_SPINLOCKS 1

#define ETHR_NATIVE_SPINLOCK_IMPL "pthread"

typedef pthread_spinlock_t ethr_native_spinlock_t;

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

static ETHR_INLINE void
ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
{
    int err = pthread_spin_init((pthread_spinlock_t *) lock, 0);
    if (err)
	ETHR_FATAL_ERROR__(err);
}

static ETHR_INLINE int
ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
{
    return pthread_spin_destroy((pthread_spinlock_t *) lock);
}

static ETHR_INLINE void
ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
{
    int err = pthread_spin_unlock((pthread_spinlock_t *) lock);
    if (err)
	ETHR_FATAL_ERROR__(err);
}

static ETHR_INLINE void
ethr_native_spin_lock(ethr_native_spinlock_t *lock)
{
    int err = pthread_spin_lock((pthread_spinlock_t *) lock);
    if (err)
	ETHR_FATAL_ERROR__(err);
}

#endif
# 82 "../include/internal/ethr_optimized_fallbacks.h"

#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
# 84 "../include/internal/ethr_optimized_fallbacks.h"
/* --- Native spinlocks using native atomics -------------------------------- */
#define ETHR_HAVE_NATIVE_SPINLOCKS 1

#define ETHR_NATIVE_SPINLOCK_IMPL "native-atomics"

typedef ethr_atomic32_t ethr_native_spinlock_t;

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

#undef ETHR_NSPN_AOP__
#define ETHR_NSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)

static ETHR_INLINE void
ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
{
    ETHR_NSPN_AOP__(init)(lock, 0);
}

static ETHR_INLINE int
ethr_native_spinlock_destroy(ethr_native_spinlock_t *lock)
{
    return ETHR_NSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY;
}

static ETHR_INLINE void
ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
{
    ETHR_ASSERT(ETHR_NSPN_AOP__(read)(lock) == 1);
    ETHR_NSPN_AOP__(set_relb)(lock, 0);
}

static ETHR_INLINE void
ethr_native_spin_lock(ethr_native_spinlock_t *lock)
{
    while (ETHR_NSPN_AOP__(cmpxchg_acqb)(lock, 1, 0) != 0) {
	while (ETHR_NSPN_AOP__(read)(lock) != 0)
	    ETHR_SPIN_BODY;
    }
    ETHR_COMPILER_BARRIER;
}

#undef ETHR_NSPN_AOP__

#endif
# 128 "../include/internal/ethr_optimized_fallbacks.h"

#endif
# 130 "../include/internal/ethr_optimized_fallbacks.h"


#if defined(ETHR_HAVE_NATIVE_RWSPINLOCKS)

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

static ETHR_INLINE int
ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock)
{
    return 0;
}

#endif
# 143 "../include/internal/ethr_optimized_fallbacks.h"

#elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
# 145 "../include/internal/ethr_optimized_fallbacks.h"
/* --- Native rwspinlocks using native atomics ------------------------------ */
#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1
#define ETHR_NATIVE_RWSPINLOCK_IMPL "native-atomics"

typedef ethr_atomic32_t ethr_native_rwlock_t;
#  define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30)

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

#undef ETHR_NRWSPN_AOP__
#define ETHR_NRWSPN_AOP__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X)

static ETHR_INLINE void
ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
{
    ETHR_NRWSPN_AOP__(init)(lock, 0);
}

static ETHR_INLINE int
ethr_native_rwlock_destroy(ethr_native_rwlock_t *lock)
{
    return ETHR_NRWSPN_AOP__(read)(lock) == 0 ? 0 : EBUSY;
}

static ETHR_INLINE void
ethr_native_read_unlock(ethr_native_rwlock_t *lock)
{
    ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) >= 0);
    ETHR_NRWSPN_AOP__(dec_relb)(lock);
}

static ETHR_INLINE void
ethr_native_read_lock(ethr_native_rwlock_t *lock)
{
    ethr_sint32_t act, exp = 0;
    while (1) {
	act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp+1, exp);
	if (act == exp)
	    break;
	/* Wait for writer to leave */
	while (act & ETHR_WLOCK_FLAG__) {
	    ETHR_SPIN_BODY;
	    act = ETHR_NRWSPN_AOP__(read)(lock);
	}
	exp = act;
    }
    ETHR_COMPILER_BARRIER;
}

static ETHR_INLINE void
ethr_native_write_unlock(ethr_native_rwlock_t *lock)
{
    ETHR_ASSERT(ETHR_NRWSPN_AOP__(read)(lock) == ETHR_WLOCK_FLAG__);
    ETHR_NRWSPN_AOP__(set_relb)(lock, 0);
}

static ETHR_INLINE void
ethr_native_write_lock(ethr_native_rwlock_t *lock)
{
    ethr_sint32_t act, exp = 0;
    while (1) {
	act = ETHR_NRWSPN_AOP__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp);
	if (act == exp)
	    break;
	/* Wait for writer to leave */
	while (act & ETHR_WLOCK_FLAG__) {
	    ETHR_SPIN_BODY;
	    act = ETHR_NRWSPN_AOP__(read)(lock);
	}
	exp = act;
    }
    act |= ETHR_WLOCK_FLAG__;
    /* Wait for readers to leave */
    while (act != ETHR_WLOCK_FLAG__) {
	ETHR_SPIN_BODY;
	act = ETHR_NRWSPN_AOP__(read_acqb)(lock);
    }
    ETHR_COMPILER_BARRIER;
}

#undef ETHR_NRWSPN_AOP__

#endif
# 228 "../include/internal/ethr_optimized_fallbacks.h"

#endif
# 230 "../include/internal/ethr_optimized_fallbacks.h"

#endif
# 232 "../include/internal/ethr_optimized_fallbacks.h"
# 466 "../include/internal/ethread.h" 2
#endif
# 467 "../include/internal/ethread.h"

typedef struct {
    void *(*thread_create_prepare_func)(void);
    void (*thread_create_parent_func)(void *);
    void (*thread_create_child_func)(void *);
} ethr_init_data;

#define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL}

typedef struct {
    void *(*alloc)(size_t);
    void *(*realloc)(void *, size_t);
    void (*free)(void *);
} ethr_memory_allocator;

#define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL}

typedef struct {
    ethr_memory_allocator std;
    ethr_memory_allocator sl;
    ethr_memory_allocator ll;
} ethr_memory_allocators;

#define ETHR_MEM_ALLOCS_DEF_INITER__					\
  {ETHR_MEM_ALLOC_DEF_INITER__,						\
   ETHR_MEM_ALLOC_DEF_INITER__,						\
   ETHR_MEM_ALLOC_DEF_INITER__}

typedef struct {
    ethr_memory_allocators mem;
    int reader_groups;
    int main_threads;
} ethr_late_init_data;

#define ETHR_LATE_INIT_DATA_DEFAULT_INITER				\
  {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0}

typedef struct {
    int detached;			/* boolean (default false) */
    int suggested_stack_size;		/* kilo words (default sys dependent) */
    char *name;                         /* max 14 char long (default no-name) */
#ifdef ETHR_OSE_THREADS
    U32 coreNo;
#endif
# 511 "../include/internal/ethread.h"
} ethr_thr_opts;

#if defined(ETHR_OSE_THREADS)
#define ETHR_THR_OPTS_DEFAULT_INITER {0, -1, NULL, 0}
#else
# 516 "../include/internal/ethread.h"
#define ETHR_THR_OPTS_DEFAULT_INITER {0, -1, NULL}
#endif
# 518 "../include/internal/ethread.h"

#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
#  define ETHR_NEED_SPINLOCK_PROTOTYPES__
#  define ETHR_NEED_RWSPINLOCK_PROTOTYPES__
#endif
# 523 "../include/internal/ethread.h"

int ethr_init(ethr_init_data *);
int ethr_late_init(ethr_late_init_data *);
int ethr_install_exit_handler(void (*funcp)(void));
int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *);
int ethr_thr_join(ethr_tid, void **);
int ethr_thr_detach(ethr_tid);
void ethr_thr_exit(void *);
ethr_tid ethr_self(void);
int ethr_getname(ethr_tid, char *, size_t);
void ethr_setname(char *);
int ethr_equal_tids(ethr_tid, ethr_tid);

int ethr_tsd_key_create(ethr_tsd_key *,char *);
int ethr_tsd_key_delete(ethr_tsd_key);
int ethr_tsd_set(ethr_tsd_key, void *);
void *ethr_tsd_get(ethr_tsd_key);

#ifdef ETHR_HAVE_ETHR_SIG_FUNCS
#if 0 /* expanded by -frewrite-includes */
#include <signal.h>
#endif /* expanded by -frewrite-includes */
# 542 "../include/internal/ethread.h"
# 543 "../include/internal/ethread.h"
int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset);
int ethr_sigwait(const sigset_t *set, int *sig);
int ethr_kill(const ethr_tid tid, const int sig);
#endif
# 547 "../include/internal/ethread.h"

void ethr_compiler_barrier(void);

#if defined(ETHR_HAVE_NATIVE_SPINLOCKS)
typedef ethr_native_spinlock_t ethr_spinlock_t;
#elif defined(__WIN32__)
# 553 "../include/internal/ethread.h"
typedef CRITICAL_SECTION ethr_spinlock_t;
#else
# 555 "../include/internal/ethread.h"
typedef pthread_mutex_t ethr_spinlock_t;
#endif
# 557 "../include/internal/ethread.h"

#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__
int ethr_spinlock_init(ethr_spinlock_t *);
int ethr_spinlock_destroy(ethr_spinlock_t *);
void ethr_spin_unlock(ethr_spinlock_t *);
void ethr_spin_lock(ethr_spinlock_t *);
#endif
# 564 "../include/internal/ethread.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
    ethr_native_spinlock_init(lock);
    return 0;
#elif defined(__WIN32__)
# 574 "../include/internal/ethread.h"
    if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX))
	return ethr_win_get_errno__();
    return 0;
#else
# 578 "../include/internal/ethread.h"
    return pthread_mutex_init((pthread_mutex_t *) lock, NULL);
#endif
# 580 "../include/internal/ethread.h"
}

static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
    return ethr_native_spinlock_destroy(lock);
#elif defined(__WIN32__)
# 588 "../include/internal/ethread.h"
    DeleteCriticalSection((CRITICAL_SECTION *) lock);
    return 0;
#else
# 591 "../include/internal/ethread.h"
    return pthread_mutex_destroy((pthread_mutex_t *) lock);
#endif
# 593 "../include/internal/ethread.h"
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
    ethr_native_spin_unlock(lock);
#elif defined(__WIN32__)
# 601 "../include/internal/ethread.h"
    LeaveCriticalSection((CRITICAL_SECTION *) lock);
#else
# 603 "../include/internal/ethread.h"
    int err = pthread_mutex_unlock((pthread_mutex_t *) lock);
    if (err)
	ETHR_FATAL_ERROR__(err);
#endif
# 607 "../include/internal/ethread.h"
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_SPINLOCKS
    ethr_native_spin_lock(lock);
#elif defined(__WIN32__)
# 615 "../include/internal/ethread.h"
    EnterCriticalSection((CRITICAL_SECTION *) lock);
#else
# 617 "../include/internal/ethread.h"
    int err = pthread_mutex_lock((pthread_mutex_t *) lock);
    if (err)
	ETHR_FATAL_ERROR__(err);
#endif
# 621 "../include/internal/ethread.h"
}

#endif /* ETHR_TRY_INLINE_FUNCS */
# 624 "../include/internal/ethread.h"

typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */

#if defined(ETHR_WIN32_THREADS)
#if 0 /* expanded by -frewrite-includes */
#  include "win/ethr_event.h"
#endif /* expanded by -frewrite-includes */
# 628 "../include/internal/ethread.h"
# 629 "../include/internal/ethread.h"
#elif defined(ETHR_PTHREADS)
# 630 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#  include "pthread/ethr_event.h"
#endif /* expanded by -frewrite-includes */
# 630 "../include/internal/ethread.h"
# 1 "../include/internal/pthread/ethr_event.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2009-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Author: Rickard Green
 */

#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
/* --- Linux futex implementation of ethread events ------------------------- */
#define ETHR_LINUX_FUTEX_IMPL__

#if 0 /* expanded by -frewrite-includes */
#include <sys/syscall.h>
#endif /* expanded by -frewrite-includes */
# 29 "../include/internal/pthread/ethr_event.h"
# 30 "../include/internal/pthread/ethr_event.h"
#if 0 /* expanded by -frewrite-includes */
#include <unistd.h>
#endif /* expanded by -frewrite-includes */
# 30 "../include/internal/pthread/ethr_event.h"
# 31 "../include/internal/pthread/ethr_event.h"
#if 0 /* expanded by -frewrite-includes */
#include <linux/futex.h>
#endif /* expanded by -frewrite-includes */
# 31 "../include/internal/pthread/ethr_event.h"
# 32 "../include/internal/pthread/ethr_event.h"
#if 0 /* expanded by -frewrite-includes */
#include <sys/time.h>
#endif /* expanded by -frewrite-includes */
# 32 "../include/internal/pthread/ethr_event.h"
# 33 "../include/internal/pthread/ethr_event.h"

#define ETHR_EVENT_OFF_WAITER__		((ethr_sint32_t) -1)
#define ETHR_EVENT_OFF__		((ethr_sint32_t) 1)
#define ETHR_EVENT_ON__ 		((ethr_sint32_t) 0)

#if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE)
#  define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE
#  define ETHR_FUTEX_WAKE__ FUTEX_WAKE_PRIVATE
#else
# 42 "../include/internal/pthread/ethr_event.h"
#  define ETHR_FUTEX_WAIT__ FUTEX_WAIT
#  define ETHR_FUTEX_WAKE__ FUTEX_WAKE
#endif
# 45 "../include/internal/pthread/ethr_event.h"

typedef struct {
    ethr_atomic32_t futex;
} ethr_event;

#define ETHR_FUTEX__(FTX, OP, VAL, TIMEOUT)		\
  (-1 == syscall(__NR_futex,				\
		 (void *) ethr_atomic32_addr((FTX)),	\
		 (OP),					\
		 (int) (VAL),				\
		 (TIMEOUT),				\
		 NULL,					\
		 0)					\
   ? errno : 0)

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)

static void ETHR_INLINE
ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
{
    ethr_sint32_t val;
    val = ethr_atomic32_xchg_mb(&e->futex, ETHR_EVENT_ON__);
    if (val == ETHR_EVENT_OFF_WAITER__) {
	int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1, NULL);
	if (res != 0)
	    ETHR_FATAL_ERROR__(res);
    }
}

static void ETHR_INLINE
ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
{
    ethr_atomic32_set(&e->futex, ETHR_EVENT_OFF__);
    ETHR_MEMORY_BARRIER;
}

#endif
# 82 "../include/internal/pthread/ethr_event.h"

#elif defined(ETHR_PTHREADS)
# 84 "../include/internal/pthread/ethr_event.h"
/* --- Posix mutex/cond pipe/select implementation of events ---------------- */

#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__)
#  define __DARWIN__ 1
#endif
# 89 "../include/internal/pthread/ethr_event.h"

#ifdef __DARWIN__
typedef struct ethr_event_fdsets___ ethr_event_fdsets__;
#endif
# 93 "../include/internal/pthread/ethr_event.h"

typedef struct {
    ethr_atomic32_t state;
    pthread_mutex_t mtx;
    pthread_cond_t cnd;
    int fd[2];
#ifdef __DARWIN__
    ethr_event_fdsets__ *fdsets;
#endif
# 102 "../include/internal/pthread/ethr_event.h"
} ethr_event;

#define ETHR_EVENT_OFF_WAITER_SELECT__	((ethr_sint32_t) -2)
#define ETHR_EVENT_OFF_WAITER__		((ethr_sint32_t) -1)
#define ETHR_EVENT_OFF__		((ethr_sint32_t) 1)
#define ETHR_EVENT_ON__ 		((ethr_sint32_t) 0)

#define ETHR_EVENT_IS_WAITING__(VAL) ((VAL) < 0)

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)

#ifndef ETHR_HAVE_PTHREAD_TIMED_COND_MONOTONIC
#if 0 /* expanded by -frewrite-includes */
#include <unistd.h>
#endif /* expanded by -frewrite-includes */
# 114 "../include/internal/pthread/ethr_event.h"
# 115 "../include/internal/pthread/ethr_event.h"
#if 0 /* expanded by -frewrite-includes */
#include <errno.h>
#endif /* expanded by -frewrite-includes */
# 115 "../include/internal/pthread/ethr_event.h"
# 116 "../include/internal/pthread/ethr_event.h"
#endif
# 117 "../include/internal/pthread/ethr_event.h"

static void ETHR_INLINE
ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
{
    ethr_sint32_t val;
    val = ethr_atomic32_xchg_mb(&e->state, ETHR_EVENT_ON__);
    if (ETHR_EVENT_IS_WAITING__(val)) {
	int res;
	if (val == ETHR_EVENT_OFF_WAITER_SELECT__) {
	    ssize_t wres;
	    int fd = e->fd[1];
	    ETHR_ASSERT(fd >= 0);
	    do {
		wres = write(fd, "!", 1);
	    } while (wres < 0 && errno == EINTR);
	    if (wres < 0 && errno != EAGAIN && errno != EWOULDBLOCK)
		ETHR_FATAL_ERROR__(errno);
	}
	else {
	    res = pthread_mutex_lock(&e->mtx);
	    if (res != 0)
		ETHR_FATAL_ERROR__(res);
	    res = pthread_cond_signal(&e->cnd);
	    if (res != 0)
		ETHR_FATAL_ERROR__(res);
	    res = pthread_mutex_unlock(&e->mtx);
	    if (res != 0)
		ETHR_FATAL_ERROR__(res);
	}
    }
}

static void ETHR_INLINE
ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
{
    ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__);
    ETHR_MEMORY_BARRIER;
}

#endif
# 157 "../include/internal/pthread/ethr_event.h"

#endif
# 159 "../include/internal/pthread/ethr_event.h"

int ethr_event_init(ethr_event *e);
int ethr_event_prepare_timed(ethr_event *e);
int ethr_event_destroy(ethr_event *e);
int ethr_event_wait(ethr_event *e);
int ethr_event_swait(ethr_event *e, int spincount);
int ethr_event_twait(ethr_event *e, ethr_sint64_t timeout);
int ethr_event_stwait(ethr_event *e, int spincount, ethr_sint64_t timeout);
#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
void ethr_event_set(ethr_event *e);
void ethr_event_reset(ethr_event *e);
#endif
# 171 "../include/internal/pthread/ethr_event.h"
# 631 "../include/internal/ethread.h" 2
#elif defined(ETHR_OSE_THREADS)
# 632 "../include/internal/ethread.h"
#if 0 /* expanded by -frewrite-includes */
#  include "ose/ethr_event.h"
#endif /* expanded by -frewrite-includes */
# 632 "../include/internal/ethread.h"
# 633 "../include/internal/ethread.h"
#endif
# 634 "../include/internal/ethread.h"

int ethr_set_main_thr_status(int, int);
int ethr_get_main_thr_status(int *);

struct ethr_ts_event_ {
    ethr_ts_event *next;
    ethr_ts_event *prev;
    ethr_event event;
    void *udata;
    ethr_atomic32_t uaflgs;
    unsigned uflgs;
    unsigned iflgs;		/* for ethr lib only */
    short rgix;			/* for ethr lib only */
    short mtix;			/* for ethr lib only */
};

#define ETHR_TS_EV_ETHREAD	(((unsigned) 1) << 0)
#define ETHR_TS_EV_INITED	(((unsigned) 1) << 1)
#define ETHR_TS_EV_TMP		(((unsigned) 1) << 2)
#define ETHR_TS_EV_MAIN_THR	(((unsigned) 1) << 3)

int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp);
int ethr_free_ts_event__(ethr_ts_event *tsep);
int ethr_make_ts_event__(ethr_ts_event **tsepp);

#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)
ethr_ts_event *ethr_get_ts_event(void);
void ethr_leave_ts_event(ethr_ts_event *);
#endif
# 663 "../include/internal/ethread.h"

#if defined(ETHR_PTHREADS)

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)

extern pthread_key_t ethr_ts_event_key__;

static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
{
    ethr_ts_event *tsep = pthread_getspecific(ethr_ts_event_key__);
    if (!tsep) {
	int res = ethr_make_ts_event__(&tsep);
	if (res != 0)
	    ETHR_FATAL_ERROR__(res);
	ETHR_ASSERT(tsep);
    }
    return tsep;
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
{

}

#endif
# 690 "../include/internal/ethread.h"

#elif defined(ETHR_WIN32_THREADS)
# 692 "../include/internal/ethread.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)

extern DWORD ethr_ts_event_key__;

static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
{
    ethr_ts_event *tsep = TlsGetValue(ethr_ts_event_key__);
    if (!tsep) {
	int res = ethr_get_tmp_ts_event__(&tsep);
	if (res != 0)
	    ETHR_FATAL_ERROR__(res);
	ETHR_ASSERT(tsep);
    }
    return tsep;
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
{
    if (tsep->iflgs & ETHR_TS_EV_TMP) {
	int res = ethr_free_ts_event__(tsep);
	if (res != 0)
	    ETHR_FATAL_ERROR__(res);
    }
}

#endif
# 721 "../include/internal/ethread.h"

#elif  defined (ETHR_OSE_THREADS)
# 723 "../include/internal/ethread.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__)

extern ethr_tsd_key ethr_ts_event_key__;

static ETHR_INLINE ethr_ts_event *
ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void)
{
    ethr_ts_event *tsep = *(ethr_ts_event**)ose_get_ppdata(ethr_ts_event_key__);
    if (!tsep) {
	int res = ethr_get_tmp_ts_event__(&tsep);
	if (res != 0)
	    ETHR_FATAL_ERROR__(res);
	ETHR_ASSERT(tsep);
    }
    return tsep;
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep)
{
    if (tsep->iflgs & ETHR_TS_EV_TMP) {
	int res = ethr_free_ts_event__(tsep);
	if (res != 0)
	    ETHR_FATAL_ERROR__(res);
    }
}

#endif
# 752 "../include/internal/ethread.h"

#endif
# 754 "../include/internal/ethread.h"

#if 0 /* expanded by -frewrite-includes */
#include "ethr_mutex.h" /* Need atomic declarations and tse */
#endif /* expanded by -frewrite-includes */
# 755 "../include/internal/ethread.h"
# 1 "../include/internal/ethr_mutex.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Mutex, rwmutex and condition variable implementation
 * Author: Rickard Green
 */

/*
 * IMPORTANT note about ethr_cond_signal() and ethr_cond_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. We do not allow that by default in order to avoid a performance
 * penalty on some platforms.
 *
 * Mutexes and condition variables can, however, be initialized as POSIX
 * compliant. When initialized as such ethr_cond_signal(), and
 * ethr_cond_broadcast() are allowed to be called even though the associated
 * mutexes aren't locked. This will, however, incur a performance penalty on
 * some platforms.
 *
 * POSIX compliant mutexes and condition variables *need* to be used together.
 */

#ifndef ETHR_MUTEX_H__
#define ETHR_MUTEX_H__

#define ETHR_RWMUTEX_INITIALIZED 	0x99999999
#define ETHR_MUTEX_INITIALIZED		0x77777777
#define ETHR_COND_INITIALIZED		0x55555555

#if 0
#  define ETHR_MTX_HARD_DEBUG
#endif
# 53 "../include/internal/ethr_mutex.h"

#if 0
#  define ETHR_MTX_CHK_EXCL
#if 1
#    define ETHR_MTX_CHK_NON_EXCL
#endif
# 59 "../include/internal/ethr_mutex.h"
#endif
# 60 "../include/internal/ethr_mutex.h"

/* #define ETHR_DBG_WIN_MTX_WITH_PTHREADS */
#ifdef ETHR_DBG_WIN_MTX_WITH_PTHREADS
typedef pthread_mutex_t CRITICAL_SECTION;
int TryEnterCriticalSection(CRITICAL_SECTION *);
void EnterCriticalSection(CRITICAL_SECTION *);
void LeaveCriticalSection(CRITICAL_SECTION *);
#endif
# 68 "../include/internal/ethr_mutex.h"

#ifdef ETHR_MTX_HARD_DEBUG
#  ifdef __GNUC__
#    warning ETHR_MTX_HARD_DEBUG
#  endif
# 73 "../include/internal/ethr_mutex.h"
/*#  define ETHR_MTX_HARD_DEBUG_LFS*/
/*#  define ETHR_MTX_HARD_DEBUG_FENCE*/
/*#  define ETHR_MTX_HARD_DEBUG_Q*/
#  define ETHR_MTX_HARD_DEBUG_WSQ

#  if !defined(ETHR_MTX_HARD_DEBUG_WSQ) && defined(ETHR_MTX_HARD_DEBUG_Q)
#    define ETHR_MTX_HARD_DEBUG_WSQ
#  endif
# 81 "../include/internal/ethr_mutex.h"
#endif
# 82 "../include/internal/ethr_mutex.h"

#ifndef ETHR_INLINE_MTX_FUNC_NAME_
#  define ETHR_INLINE_MTX_FUNC_NAME_(X) X
#endif
# 86 "../include/internal/ethr_mutex.h"

#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)

#ifdef ETHR_DEBUG
#  ifndef ETHR_MTX_CHK_EXCL
#    define ETHR_MTX_CHK_EXCL
#  endif
# 93 "../include/internal/ethr_mutex.h"
#  ifndef ETHR_MTX_CHK_NON_EXCL
#    define ETHR_MTX_CHK_NON_EXCL
#  endif
# 96 "../include/internal/ethr_mutex.h"
#endif
# 97 "../include/internal/ethr_mutex.h"

#if 0
#  define ETHR_MTX_Q_LOCK_SPINLOCK__
#  define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t
#elif defined(ETHR_PTHREADS) || defined(ETHR_OSE_THREADS)
# 102 "../include/internal/ethr_mutex.h"
#  define ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__
#  define ETHR_MTX_QLOCK_TYPE__ pthread_mutex_t
#elif defined(ETHR_WIN32_THREADS)
# 105 "../include/internal/ethr_mutex.h"
#  define ETHR_MTX_Q_LOCK_CRITICAL_SECTION__
#  define ETHR_MTX_QLOCK_TYPE__ CRITICAL_SECTION
#else
# 108 "../include/internal/ethr_mutex.h"
#  error Need a qlock implementation
#endif
# 110 "../include/internal/ethr_mutex.h"

#define ETHR_RWMTX_W_FLG__		(((ethr_sint32_t) 1) << 31)
#define ETHR_RWMTX_W_WAIT_FLG__		(((ethr_sint32_t) 1) << 30)
#define ETHR_RWMTX_R_WAIT_FLG__		(((ethr_sint32_t) 1) << 29)

/* frequent read kind */
#define ETHR_RWMTX_R_FLG__		(((ethr_sint32_t) 1) << 28)
#define ETHR_RWMTX_R_ABRT_UNLCK_FLG__	(((ethr_sint32_t) 1) << 27)
#define ETHR_RWMTX_R_PEND_UNLCK_MASK__	(ETHR_RWMTX_R_ABRT_UNLCK_FLG__ - 1)

/* normal kind */
#define ETHR_RWMTX_RS_MASK__		(ETHR_RWMTX_R_WAIT_FLG__ - 1)

#define ETHR_RWMTX_WAIT_FLGS__ \
  (ETHR_RWMTX_W_WAIT_FLG__|ETHR_RWMTX_R_WAIT_FLG__)

#define ETHR_CND_WAIT_FLG__		ETHR_RWMTX_R_WAIT_FLG__

#ifdef ETHR_DEBUG
#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V)			\
  ETHR_ASSERT(!((V) & ~(ETHR_RWMTX_W_FLG__		\
			| ETHR_RWMTX_W_WAIT_FLG__	\
			| ETHR_RWMTX_R_WAIT_FLG__	\
			| ETHR_RWMTX_RS_MASK__)))
#else
# 135 "../include/internal/ethr_mutex.h"
#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V)
#endif
# 137 "../include/internal/ethr_mutex.h"

#define ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(MTX)		\
  ETHR_DBG_CHK_UNUSED_FLG_BITS(ethr_atomic32_read(&(MTX)->mtxb.flgs))

struct ethr_mutex_base_ {
#ifdef ETHR_MTX_HARD_DEBUG_FENCE
    long pre_fence;
#endif
# 145 "../include/internal/ethr_mutex.h"
    ethr_atomic32_t flgs;
    short aux_scnt;
    short main_scnt;
    ETHR_MTX_QLOCK_TYPE__ qlck;
    ethr_ts_event *q;
#ifdef ETHR_MTX_HARD_DEBUG_WSQ
    int ws;
#endif
# 153 "../include/internal/ethr_mutex.h"
#ifdef ETHR_MTX_CHK_EXCL
    ethr_atomic32_t exclusive;
#endif
# 156 "../include/internal/ethr_mutex.h"
#ifdef ETHR_MTX_CHK_NON_EXCL
    ethr_atomic32_t non_exclusive;
#endif
# 159 "../include/internal/ethr_mutex.h"
#ifdef ETHR_MTX_HARD_DEBUG_LFS
    ethr_atomic32_t hdbg_lfs;
#endif
# 162 "../include/internal/ethr_mutex.h"
};

#endif
# 165 "../include/internal/ethr_mutex.h"

typedef struct {
    int main_spincount;
    int aux_spincount;
    int posix_compliant;
} ethr_mutex_opt;

#define ETHR_MUTEX_OPT_DEFAULT_INITER {-1, -1, 0}

typedef struct {
    int main_spincount;
    int aux_spincount;
    int posix_compliant;
} ethr_cond_opt;

#define ETHR_COND_OPT_DEFAULT_INITER {-1, -1, 0}

#ifdef ETHR_USE_OWN_MTX_IMPL__

typedef struct ethr_mutex_ ethr_mutex;
struct ethr_mutex_ {
    struct ethr_mutex_base_ mtxb;
#ifdef ETHR_MTX_HARD_DEBUG_FENCE
    long post_fence;
#endif
# 190 "../include/internal/ethr_mutex.h"
#if ETHR_XCHK
    int initialized;
#endif
# 193 "../include/internal/ethr_mutex.h"
};

typedef struct ethr_cond_ ethr_cond;
struct ethr_cond_ {
#ifdef ETHR_MTX_HARD_DEBUG_FENCE
    struct {
	long pre_fence;
    } mtxb; /* mtxb allows us to use same macro as for mutex and rwmutex... */
#endif
# 202 "../include/internal/ethr_mutex.h"
    ETHR_MTX_QLOCK_TYPE__ qlck;
    ethr_ts_event *q;
    short aux_scnt;
    short main_scnt;
#ifdef ETHR_MTX_HARD_DEBUG_FENCE
    long post_fence;
#endif
# 209 "../include/internal/ethr_mutex.h"
#if ETHR_XCHK
    int initialized;
#endif
# 212 "../include/internal/ethr_mutex.h"
};

#elif (defined(ETHR_PTHREADS) || defined(ETHR_OSE_THREADS)) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
# 215 "../include/internal/ethr_mutex.h"

typedef struct ethr_mutex_ ethr_mutex;
struct ethr_mutex_ {
    pthread_mutex_t pt_mtx;
#if ETHR_XCHK
    int initialized;
#endif
# 222 "../include/internal/ethr_mutex.h"
};

typedef struct ethr_cond_ ethr_cond;
struct ethr_cond_ {
    pthread_cond_t pt_cnd;
#if ETHR_XCHK
    int initialized;
#endif
# 230 "../include/internal/ethr_mutex.h"
};

#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
# 233 "../include/internal/ethr_mutex.h"
#  define ETHR_WIN_MUTEX__

typedef struct ethr_mutex_ ethr_mutex;
struct ethr_mutex_ {
    int posix_compliant;
    CRITICAL_SECTION cs;
    ethr_ts_event *wakeups;
    ethr_atomic32_t have_wakeups; /* only when posix compliant */
    ethr_atomic32_t locked;       /* only when posix compliant */
    ethr_spinlock_t lock;         /* only when posix compliant */
#if ETHR_XCHK
    int initialized;
#endif
# 246 "../include/internal/ethr_mutex.h"
};

typedef struct ethr_cond_ ethr_cond;
struct ethr_cond_ {
    int posix_compliant;
    CRITICAL_SECTION cs;
    ethr_ts_event *waiters;
    int spincount;
#if ETHR_XCHK
    int initialized;
#endif
# 257 "../include/internal/ethr_mutex.h"
};

#else
# 260 "../include/internal/ethr_mutex.h"
#  error "no mutex implementation"
#endif
# 262 "../include/internal/ethr_mutex.h"

int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *);
int ethr_mutex_init(ethr_mutex *);
int ethr_mutex_destroy(ethr_mutex *);
#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)
int ethr_mutex_trylock(ethr_mutex *);
void ethr_mutex_lock(ethr_mutex *);
void ethr_mutex_unlock(ethr_mutex *);
#endif
# 271 "../include/internal/ethr_mutex.h"
int ethr_cond_init_opt(ethr_cond *, ethr_cond_opt *);
int ethr_cond_init(ethr_cond *);
int ethr_cond_destroy(ethr_cond *);
void ethr_cond_signal(ethr_cond *);
void ethr_cond_broadcast(ethr_cond *);
int ethr_cond_wait(ethr_cond *, ethr_mutex *);

typedef enum {
    ETHR_RWMUTEX_TYPE_NORMAL,
    ETHR_RWMUTEX_TYPE_FREQUENT_READ,
    ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
} ethr_rwmutex_type;

typedef enum {
    ETHR_RWMUTEX_LONG_LIVED,
    ETHR_RWMUTEX_SHORT_LIVED,
    ETHR_RWMUTEX_UNKNOWN_LIVED
} ethr_rwmutex_lived;

typedef struct {
    ethr_rwmutex_type type;
    ethr_rwmutex_lived lived;
    int main_spincount;
    int aux_spincount;
} ethr_rwmutex_opt;

#define ETHR_RWMUTEX_OPT_DEFAULT_INITER \
  {ETHR_RWMUTEX_TYPE_NORMAL, ETHR_RWMUTEX_UNKNOWN_LIVED, -1, -1}

#ifdef ETHR_USE_OWN_RWMTX_IMPL__

typedef union {
    struct {
	ethr_atomic32_t readers;
	int waiting_readers;
	int byte_offset;
	ethr_rwmutex_lived lived;
    } data;
    char align__[ETHR_CACHE_LINE_SIZE];
} ethr_rwmtx_readers_array__;

typedef struct ethr_rwmutex_ ethr_rwmutex;
struct ethr_rwmutex_ {
    struct ethr_mutex_base_ mtxb;
    ethr_rwmutex_type type;
    ethr_ts_event *rq_end;
    union {
	ethr_rwmtx_readers_array__ *ra;
	int rs;
    } tdata;
#ifdef ETHR_MTX_HARD_DEBUG_FENCE
    long post_fence;
#endif
# 324 "../include/internal/ethr_mutex.h"
#if ETHR_XCHK
    int initialized;
#endif
# 327 "../include/internal/ethr_mutex.h"
};

#else /* pthread_rwlock */
# 330 "../include/internal/ethr_mutex.h"

typedef struct ethr_rwmutex_ ethr_rwmutex;
struct ethr_rwmutex_ {
    pthread_rwlock_t pt_rwlock;
#if ETHR_XCHK
    int initialized;
#endif
# 337 "../include/internal/ethr_mutex.h"
};

#endif /* pthread_rwlock */
# 340 "../include/internal/ethr_mutex.h"

int ethr_rwmutex_set_reader_group(int);
int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *);
int ethr_rwmutex_init(ethr_rwmutex *);
int ethr_rwmutex_destroy(ethr_rwmutex *);
#if defined(ETHR_USE_OWN_RWMTX_IMPL__) \
    || !defined(ETHR_TRY_INLINE_FUNCS) \
    || defined(ETHR_MUTEX_IMPL__)
int ethr_rwmutex_tryrlock(ethr_rwmutex *);
void ethr_rwmutex_rlock(ethr_rwmutex *);
void ethr_rwmutex_runlock(ethr_rwmutex *);
int ethr_rwmutex_tryrwlock(ethr_rwmutex *);
void ethr_rwmutex_rwlock(ethr_rwmutex *);
void ethr_rwmutex_rwunlock(ethr_rwmutex *);
#endif
# 355 "../include/internal/ethr_mutex.h"

#ifdef ETHR_MTX_HARD_DEBUG
#define ETHR_MTX_HARD_ASSERT(A) \
  ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__,#A)))
#else
# 360 "../include/internal/ethr_mutex.h"
#define ETHR_MTX_HARD_ASSERT(A) ((void) 1)
#endif
# 362 "../include/internal/ethr_mutex.h"

#ifdef ETHR_MTX_HARD_DEBUG_LFS
#  define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB)				\
do {									\
    ethr_atomic32_init(&(MTXB)->hdbg_lfs, 0);				\
} while (0)
#  define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB)				\
do {									\
    ethr_sint32_t val__;						\
    ETHR_COMPILER_BARRIER;						\
    val__ = ethr_atomic32_inc_read(&(MTXB)->hdbg_lfs);			\
    ETHR_MTX_HARD_ASSERT(val__ > 0);					\
} while (0)
#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES)			\
do {									\
    ETHR_COMPILER_BARRIER;						\
    if ((RES) == 0)							\
	ETHR_MTX_HARD_DEBUG_LFS_RLOCK((MTXB));				\
    else								\
	ETHR_MTX_HARD_ASSERT((RES) == EBUSY);				\
} while (0)
#  define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB)				\
do {									\
    ethr_sint32_t val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs);	\
    ETHR_MTX_HARD_ASSERT(val__ >= 0);					\
    ETHR_COMPILER_BARRIER;						\
} while (0)
#  define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB)				\
do {									\
    ethr_sint32_t val__;						\
    ETHR_COMPILER_BARRIER;						\
    val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs);			\
    ETHR_MTX_HARD_ASSERT(val__ == -1);					\
} while (0)
#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES)			\
do {									\
    ETHR_COMPILER_BARRIER;						\
    if ((RES) == 0)							\
	ETHR_MTX_HARD_DEBUG_LFS_RWLOCK((MTXB));				\
    else								\
	ETHR_MTX_HARD_ASSERT((RES) == EBUSY);				\
} while (0)
#  define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB)			\
do {									\
    ethr_sint32_t val__ = ethr_atomic32_inctest(&(MTXB)->hdbg_lfs);	\
    ETHR_MTX_HARD_ASSERT(val__ == 0);					\
    ETHR_COMPILER_BARRIER;						\
} while (0)
#else
# 411 "../include/internal/ethr_mutex.h"
#  define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB)
#  define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB)
#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES)
#  define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB)
#  define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB)
#  define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES)
#  define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB)
#endif
# 419 "../include/internal/ethr_mutex.h"

#ifdef ETHR_MTX_HARD_DEBUG_FENCE

#if ETHR_SIZEOF_PTR == 8
#  define ETHR_MTX_HARD_DEBUG_PRE_FENCE		0xdeadbeefdeadbeefL
#  define ETHR_MTX_HARD_DEBUG_POST_FENCE	0xdeaddeaddeaddeadL
#else
# 426 "../include/internal/ethr_mutex.h"
#  define ETHR_MTX_HARD_DEBUG_PRE_FENCE		0xdeaddeadL
#  define ETHR_MTX_HARD_DEBUG_POST_FENCE	0xdeaddeadL
#endif
# 429 "../include/internal/ethr_mutex.h"

#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) \
do { \
    ETHR_COMPILER_BARRIER; \
    ETHR_MTX_HARD_ASSERT((X)->mtxb.pre_fence == ETHR_MTX_HARD_DEBUG_PRE_FENCE);\
    ETHR_MTX_HARD_ASSERT((X)->post_fence == ETHR_MTX_HARD_DEBUG_POST_FENCE); \
    ETHR_COMPILER_BARRIER; \
} while (0)
#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) \
do { \
    (X)->mtxb.pre_fence = ETHR_MTX_HARD_DEBUG_PRE_FENCE; \
    (X)->post_fence = ETHR_MTX_HARD_DEBUG_POST_FENCE; \
} while (0)
#else
# 443 "../include/internal/ethr_mutex.h"
#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X)
#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X)
#endif
# 446 "../include/internal/ethr_mutex.h"

#ifdef ETHR_MTX_CHK_EXCL

#if !defined(ETHR_DEBUG) && defined(__GNUC__)
#warning "check exclusive is enabled"
#endif
# 452 "../include/internal/ethr_mutex.h"

#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)		\
    ethr_atomic32_init(&(MTXB)->exclusive, 0)

#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)		\
do {							\
    ETHR_COMPILER_BARRIER;				\
    if (!ethr_atomic32_read(&(MTXB)->exclusive))	\
	ethr_assert_failed(__FILE__, __LINE__, __func__,\
			   "is exclusive");		\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)		\
do {							\
    ETHR_COMPILER_BARRIER;				\
    if (ethr_atomic32_read(&(MTXB)->exclusive))		\
	ethr_assert_failed(__FILE__, __LINE__, __func__,\
			   "is not exclusive");		\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)		\
do {							\
    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL((MTXB));		\
    ethr_atomic32_set(&(MTXB)->exclusive, 1);		\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)		\
do {							\
    ETHR_MTX_CHK_EXCL_IS_EXCL((MTXB));			\
    ethr_atomic32_set(&(MTXB)->exclusive, 0);		\
    ETHR_COMPILER_BARRIER;				\
} while (0)

#ifdef ETHR_MTX_CHK_NON_EXCL

#if !defined(ETHR_DEBUG) && defined(__GNUC__)
#warning "check non-exclusive is enabled"
#endif
# 490 "../include/internal/ethr_mutex.h"

#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)		\
    ethr_atomic32_init(&(MTXB)->non_exclusive, 0)
#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)		\
do {							\
    ETHR_COMPILER_BARRIER;				\
    if (!ethr_atomic32_read(&(MTXB)->non_exclusive))	\
	ethr_assert_failed(__FILE__, __LINE__, __func__,\
			   "is non-exclusive");		\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)	\
do {							\
    ETHR_COMPILER_BARRIER;				\
    if (ethr_atomic32_read(&(MTXB)->non_exclusive))	\
	ethr_assert_failed(__FILE__, __LINE__, __func__,\
			   "is not non-exclusive");	\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)	\
do {							\
    ETHR_COMPILER_BARRIER;				\
    ethr_atomic32_inc(&(MTXB)->non_exclusive);		\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)	\
do {							\
    ETHR_COMPILER_BARRIER;				\
    ethr_atomic32_add(&(MTXB)->non_exclusive, (NO));	\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)	\
do {							\
    ETHR_COMPILER_BARRIER;				\
    ethr_atomic32_dec(&(MTXB)->non_exclusive);		\
    ETHR_COMPILER_BARRIER;				\
} while (0)
#else
# 528 "../include/internal/ethr_mutex.h"
#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
#endif
# 535 "../include/internal/ethr_mutex.h"

#else
# 537 "../include/internal/ethr_mutex.h"
#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)
#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)
#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)
#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)
#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)
#  define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
#  define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
#  define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
#  define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
#endif
# 549 "../include/internal/ethr_mutex.h"

#  define ETHR_MTX_CHK_EXCL_INIT(MTXB)			\
do {							\
    ETHR_MTX_CHK_EXCL_INIT__((MTXB));			\
    ETHR_MTX_CHK_NON_EXCL_INIT__((MTXB));		\
} while (0)


#ifdef ETHR_USE_OWN_MTX_IMPL__

#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50
#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50

#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0
#define ETHR_CND_DEFAULT_AUX_SPINCOUNT 0

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)

void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t);
void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t);

static ETHR_INLINE int
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
{
    ethr_sint32_t act;
    int res;
    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);

    act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
    res = (act == 0) ? 0 : EBUSY;

#ifdef ETHR_MTX_CHK_EXCL
    if (res == 0)
	ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
#endif
# 587 "../include/internal/ethr_mutex.h"

    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res);
    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);

    ETHR_COMPILER_BARRIER;
    return res;
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
{
    ethr_sint32_t act;
    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);

    act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
    if (act != 0)
	ethr_mutex_lock_wait__(mtx, act);

    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);

    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);

    ETHR_COMPILER_BARRIER;
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
{
    ethr_sint32_t act;
    ETHR_COMPILER_BARRIER;
    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb);
    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);

    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&mtx->mtxb);

    act = ethr_atomic32_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__);
    if (act != ETHR_RWMTX_W_FLG__)
	ethr_mutex_unlock_wake__(mtx, act);

    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
    ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx);
}

#endif /* ETHR_TRY_INLINE_FUNCS */
# 636 "../include/internal/ethr_mutex.h"

#elif (defined(ETHR_PTHREADS) || defined(ETHR_OSE_THREADS)) && !defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
# 638 "../include/internal/ethr_mutex.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)

static ETHR_INLINE int
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
{
    int res;
    res = pthread_mutex_trylock(&mtx->pt_mtx);
    if (res != 0 && res != EBUSY)
	ETHR_FATAL_ERROR__(res);
    return res;
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
{
    int res = pthread_mutex_lock(&mtx->pt_mtx);
    if (res != 0)
	ETHR_FATAL_ERROR__(res);
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
{
    int res = pthread_mutex_unlock(&mtx->pt_mtx);
    if (res != 0)
	ETHR_FATAL_ERROR__(res);
}

#endif /* ETHR_TRY_INLINE_FUNCS */
# 668 "../include/internal/ethr_mutex.h"

#elif defined(ETHR_WIN32_THREADS) || defined(ETHR_DBG_WIN_MTX_WITH_PTHREADS)
# 670 "../include/internal/ethr_mutex.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)

static ETHR_INLINE int
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
{
    if (!TryEnterCriticalSection(&mtx->cs))
	return EBUSY;
    if (mtx->posix_compliant)
	ethr_atomic32_set(&mtx->locked, 1);
    return 0;
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
{
    EnterCriticalSection(&mtx->cs);
    if (mtx->posix_compliant)
	ethr_atomic32_set(&mtx->locked, 1);
}

void ethr_mutex_cond_wakeup__(ethr_mutex *mtx);

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
{
    if (mtx->posix_compliant) {
	ethr_atomic32_set_mb(&mtx->locked, 0);
	if (ethr_atomic32_read_acqb(&mtx->have_wakeups))
	    goto cond_wakeup;
	else
	    goto leave_cs;
    }

    if (mtx->wakeups) {
    cond_wakeup:
	ethr_mutex_cond_wakeup__(mtx);
    }
    else {
    leave_cs:
	LeaveCriticalSection(&mtx->cs);
    }
}

#endif /* ETHR_TRY_INLINE_FUNCS */
# 715 "../include/internal/ethr_mutex.h"

#endif
# 717 "../include/internal/ethr_mutex.h"

#ifdef ETHR_USE_OWN_RWMTX_IMPL__

#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50
#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50

#else /* pthread_rwlock */
# 726 "../include/internal/ethr_mutex.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__)

static ETHR_INLINE int
ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx)
{
    int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock);
    if (res != 0 && res != EBUSY)
	ETHR_FATAL_ERROR__(res);
    return res;
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx)
{
    int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock);
    if (res != 0)
	ETHR_FATAL_ERROR__(res);
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx)
{
    int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
    if (res != 0)
	ETHR_FATAL_ERROR__(res);
}

static ETHR_INLINE int
ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx)
{
    int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock);
    if (res != 0 && res != EBUSY)
	ETHR_FATAL_ERROR__(res);
    return res;
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx)
{
    int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock);
    if (res != 0)
	ETHR_FATAL_ERROR__(res);
}

static ETHR_INLINE void
ETHR_INLINE_MTX_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx)
{
    int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock);
    if (res != 0)
	ETHR_FATAL_ERROR__(res);
}

#endif /* ETHR_TRY_INLINE_FUNCS */
# 780 "../include/internal/ethr_mutex.h"

#endif /* pthread_rwlock */
# 782 "../include/internal/ethr_mutex.h"

int ethr_mutex_lib_init(int);
int ethr_mutex_lib_late_init(int, int);

#endif /* #ifndef ETHR_MUTEX_H__ */
# 787 "../include/internal/ethr_mutex.h"
# 756 "../include/internal/ethread.h" 2

#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
typedef ethr_native_rwlock_t ethr_rwlock_t;
#else
# 760 "../include/internal/ethread.h"
typedef ethr_rwmutex ethr_rwlock_t;
#endif
# 762 "../include/internal/ethread.h"

#ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__
int ethr_rwlock_init(ethr_rwlock_t *);
int ethr_rwlock_destroy(ethr_rwlock_t *);
void ethr_read_unlock(ethr_rwlock_t *);
void ethr_read_lock(ethr_rwlock_t *);
void ethr_write_unlock(ethr_rwlock_t *);
void ethr_write_lock(ethr_rwlock_t *);
#endif
# 771 "../include/internal/ethread.h"

#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)

static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
    ethr_native_rwlock_init(lock);
    return 0;
#else
# 781 "../include/internal/ethread.h"
    return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL);
#endif
# 783 "../include/internal/ethread.h"
}

static ETHR_INLINE int
ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
    return ethr_native_rwlock_destroy(lock);
#else
# 791 "../include/internal/ethread.h"
    return ethr_rwmutex_destroy((ethr_rwmutex *) lock);
#endif
# 793 "../include/internal/ethread.h"
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
    ethr_native_read_unlock(lock);
#else
# 801 "../include/internal/ethread.h"
    ethr_rwmutex_runlock((ethr_rwmutex *) lock);
#endif
# 803 "../include/internal/ethread.h"
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
    ethr_native_read_lock(lock);
#else
# 811 "../include/internal/ethread.h"
    ethr_rwmutex_rlock((ethr_rwmutex *) lock);
#endif
# 813 "../include/internal/ethread.h"
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
    ethr_native_write_unlock(lock);
#else
# 821 "../include/internal/ethread.h"
    ethr_rwmutex_rwunlock((ethr_rwmutex *) lock);
#endif
# 823 "../include/internal/ethread.h"
}

static ETHR_INLINE void
ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock)
{
#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS
    ethr_native_write_lock(lock);
#else
# 831 "../include/internal/ethread.h"
    ethr_rwmutex_rwlock((ethr_rwmutex *) lock);
#endif
# 833 "../include/internal/ethread.h"
}

#endif /* ETHR_TRY_INLINE_FUNCS */
# 836 "../include/internal/ethread.h"

#endif /* #ifndef ETHREAD_H__ */
# 838 "../include/internal/ethread.h"
# 266 "beam/erl_threads.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 266 "beam/erl_threads.h"
# 1 "beam/erl_lock_check.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2005-2012. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
 * Description: A lock checker that checks that each thread acquires
 *              locks according to a predefined global lock order. The
 *              global lock order is used to prevent deadlocks. If the
 *              lock order is violated, an error message is printed
 *              and the emulator aborts. The lock checker is only
 *              intended to be enabled when debugging.
 *
 * Author: Rickard Green
 */

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_lock_check.h"
# 1 "beam/sys.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __SYS_H__
#define __SYS_H__

#ifdef ERTS_INLINE
#  ifndef ERTS_CAN_INLINE
#    define ERTS_CAN_INLINE 1
#  endif
# 28 "beam/sys.h"
#else
# 29 "beam/sys.h"
#  if defined(__GNUC__)
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline__
#  elif defined(__WIN32__)
# 33 "beam/sys.h"
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline
#  else
# 36 "beam/sys.h"
#    define ERTS_CAN_INLINE 0
#    define ERTS_INLINE
#  endif
# 39 "beam/sys.h"
#endif
# 40 "beam/sys.h"

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#  undef ERTS_CAN_INLINE
#  define ERTS_CAN_INLINE 0
#  undef ERTS_INLINE
#  define ERTS_INLINE
#endif
# 47 "beam/sys.h"

#if ERTS_CAN_INLINE
#define ERTS_GLB_INLINE static ERTS_INLINE
#else
# 51 "beam/sys.h"
#define ERTS_GLB_INLINE
#endif
# 53 "beam/sys.h"

#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
#else
# 57 "beam/sys.h"
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
# 59 "beam/sys.h"

#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
#  define NO_FPE_SIGNALS
#endif
# 63 "beam/sys.h"

#ifdef DISABLE_CHILD_WAITER_THREAD
#undef ENABLE_CHILD_WAITER_THREAD
#endif
# 67 "beam/sys.h"

#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
#undef ENABLE_CHILD_WAITER_THREAD
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
# 72 "beam/sys.h"

#define ERTS_I64_LITERAL(X) X##LL

#if defined (__WIN32__)
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_sys.h"
#endif /* expanded by -frewrite-includes */
# 76 "beam/sys.h"
# 77 "beam/sys.h"
#elif defined (__OSE__)
# 78 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_ose_sys.h"
#endif /* expanded by -frewrite-includes */
# 78 "beam/sys.h"
# 79 "beam/sys.h"
#else 
# 80 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_unix_sys.h"
#endif /* expanded by -frewrite-includes */
# 80 "beam/sys.h"
# 81 "beam/sys.h"
#ifndef UNIX
#  define UNIX 1
#endif
# 84 "beam/sys.h"
#endif
# 85 "beam/sys.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_misc_utils.h"
#endif /* expanded by -frewrite-includes */
# 86 "beam/sys.h"
# 87 "beam/sys.h"

/*
 * To allow building of Universal Binaries for Mac OS X,
 * we must not depend on the endian detected by the configure script.
 */
#if defined(__APPLE__)
#  if defined(__BIG_ENDIAN__) && !defined(WORDS_BIGENDIAN)
#    define WORDS_BIGENDIAN 1
#  elif !defined(__BIG_ENDIAN__) && defined(WORDS_BIGENDIAN)
# 96 "beam/sys.h"
#    undef WORDS_BIGENDIAN
#  endif
# 98 "beam/sys.h"
#endif
# 99 "beam/sys.h"

/*
 * Make sure we have a type for FD's (used by erl_check_io)
 */

#ifndef ERTS_SYS_FD_TYPE
#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
typedef int ErtsSysFdType;
#else
# 108 "beam/sys.h"
#ifndef ERTS_SYS_FD_INVALID
# error missing ERTS_SYS_FD_INVALID
#endif
# 111 "beam/sys.h"
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
# 113 "beam/sys.h"

#if !defined(__GNUC__)
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 117 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 120 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 123 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 126 "beam/sys.h"

#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
#  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
#  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# 131 "beam/sys.h"
#  define ERTS_LIKELY(BOOL)   (BOOL)
#  define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
# 134 "beam/sys.h"
#ifdef __GNUC__
#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
#  else
# 138 "beam/sys.h"
#    define ERTS_DECLARE_DUMMY(X) X
#  endif
# 140 "beam/sys.h"
#else
# 141 "beam/sys.h"
#  define ERTS_DECLARE_DUMMY(X) X
#endif
# 143 "beam/sys.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 149 "beam/sys.h"
#      define __func__ __FUNCTION__
#    endif
# 151 "beam/sys.h"
#  endif
# 152 "beam/sys.h"
#endif
# 153 "beam/sys.h"

#define ERTS_MK_VSN_INT(Major, Minor, Build) \
    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))

#ifndef ERTS_EXIT_AFTER_DUMP
#  define ERTS_EXIT_AFTER_DUMP exit
#endif
# 160 "beam/sys.h"

/* In VC++, noreturn is a declspec that has to be before the types,
 * but in GNUC it is an att ribute to be placed between return type
 * and function name, hence __decl_noreturn <types> __noreturn <function name>
 *
 * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
 */
#if __GNUC__
#  define __decl_noreturn
#  ifndef __noreturn
#     define __noreturn __attribute__((noreturn))
#  endif
# 172 "beam/sys.h"
#else
# 173 "beam/sys.h"
#  if defined(__WIN32__) && defined(_MSC_VER)
#    define __noreturn
#    define __decl_noreturn __declspec(noreturn)
#  else
# 177 "beam/sys.h"
#    define __noreturn
#    define __decl_noreturn
#  endif
# 180 "beam/sys.h"
#endif
# 181 "beam/sys.h"

#define ERTS_ASSERT(e) \
    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))

__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
						 const char* file, int line);

#ifdef DEBUG
#  define ASSERT(e) ERTS_ASSERT(e)
#else
# 191 "beam/sys.h"
#  define ASSERT(e) ((void) 1)
#endif
# 193 "beam/sys.h"

/* ERTS_UNDEF can be used to silence false warnings about
 * "variable may be used uninitialized" while keeping the variable
 * marked as undefined by valgrind.
 */
#ifdef VALGRIND
#  define ERTS_UNDEF(V,I)
#else
# 201 "beam/sys.h"
#  define ERTS_UNDEF(V,I) V = I
#endif
# 203 "beam/sys.h"

/*
 * Compile time assert
 * (the actual compiler error msg can be a bit confusing)
 */
#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
# define ERTS_CT_ASSERT(e) \
    do { \
	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
    } while(0)
#else
# 214 "beam/sys.h"
# define ERTS_CT_ASSERT(e) \
    do { \
        enum { compile_time_assert__ = 1/(e) }; \
    } while (0)
#endif
# 219 "beam/sys.h"

/*
 * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
 * But MSC doesn't define __STDC__, unless we compile with the -Za
 * flag (strict ANSI C, no Microsoft extension).  Compiling with -Za
 * doesn't work: some Microsoft headers fail to compile...
 *
 * Solution: Test if __STDC__ or _MSC_VER is defined.
 *
 * Note: Simply defining __STDC__ doesn't work, as some Microsoft
 * headers will fail to compile!
 */

#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 232 "beam/sys.h"
# 233 "beam/sys.h"

/* This isn't sys-dependent, but putting it here benefits sys.c and drivers
   - allow use of 'const' regardless of compiler */

#if !defined(__STDC__) && !defined(_MSC_VER)
#  define const
#endif
# 240 "beam/sys.h"

#undef __deprecated
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
#  define __deprecated __attribute__((deprecated))
#else
# 245 "beam/sys.h"
#  define __deprecated
#endif
# 247 "beam/sys.h"
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
#  define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
#else
# 250 "beam/sys.h"
#  define erts_align_attribute(SZ)
#endif
# 252 "beam/sys.h"

/*
** Data types:
**
** Eterm: A tagged erlang term (possibly 64 bits)
** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
** UInt:  An unsigned integer exactly as large as an Eterm.
** SInt:  A signed integer exactly as large as an eterm and therefor large
**        enough to hold the return value of the signed_val() macro.
** UWord: An unsigned integer at least as large as a void * and also as large
**          or larger than an Eterm
** SWord: A signed integer at least as large as a void * and also as large
**          or larger than an Eterm
** Uint32: An unsigned integer of 32 bits exactly
** Sint32: A signed integer of 32 bits exactly
** Uint16: An unsigned integer of 16 bits exactly
** Sint16: A signed integer of 16 bits exactly.
*/

#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
#error Cannot handle this combination of int/long/void*/size_t sizes
#endif
# 276 "beam/sys.h"

#if SIZEOF_VOID_P == 8
#undef  ARCH_32
#define ARCH_64
#define ERTS_SIZEOF_TERM 8
#elif SIZEOF_VOID_P == 4
# 282 "beam/sys.h"
#define ARCH_32
#undef  ARCH_64
#define ERTS_SIZEOF_TERM 4
#else
# 286 "beam/sys.h"
#error Neither 32 nor 64 bit architecture
#endif
# 288 "beam/sys.h"
#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
#    define HALFWORD_HEAP 1
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND) ASSERT(COND)
#    undef ERTS_SIZEOF_TERM
#    define ERTS_SIZEOF_TERM 4
#else
# 295 "beam/sys.h"
#    define HALFWORD_HEAP 0
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND)
#endif
# 299 "beam/sys.h"

#if SIZEOF_VOID_P != SIZEOF_SIZE_T
#error sizeof(void*) != sizeof(size_t)
#endif
# 303 "beam/sys.h"

#if HALFWORD_HEAP

#if SIZEOF_INT == 4
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define ERTS_UINT_MAX UINT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#else
# 314 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 316 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long UWord;
typedef long          SWord;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#elif SIZEOF_VOID_P == SIZEOF_INT
# 325 "beam/sys.h"
typedef unsigned int UWord;
typedef int          SWord;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 332 "beam/sys.h"
typedef unsigned long long UWord;
typedef long long          SWord;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#else
# 339 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 341 "beam/sys.h"

#else /* !HALFWORD_HEAP */
# 343 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long Eterm;
typedef unsigned long Uint;
typedef long          Sint;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_INT
# 355 "beam/sys.h"
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 365 "beam/sys.h"
typedef unsigned long long Eterm;
typedef unsigned long long Uint;
typedef long long          Sint;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
#if defined(__WIN32__)
#define ErtsStrToSint _strtoi64
#else
# 376 "beam/sys.h"
#define ErtsStrToSint strtoll
#endif
# 378 "beam/sys.h"
#else
# 379 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 381 "beam/sys.h"

typedef Uint UWord;
typedef Sint SWord;
#define ERTS_UINT_MAX ERTS_UWORD_MAX

#endif /* HALFWORD_HEAP */
# 387 "beam/sys.h"

typedef UWord BeamInstr;

#ifndef HAVE_INT64
#  if SIZEOF_LONG == 8
#    define HAVE_INT64 1
typedef unsigned long Uint64;
typedef long          Sint64;
#    ifdef ULONG_MAX
#      define ERTS_UINT64_MAX ULONG_MAX
#    endif
# 398 "beam/sys.h"
#    ifdef LONG_MAX
#      define ERTS_SINT64_MAX LONG_MAX
#    endif
# 401 "beam/sys.h"
#    ifdef LONG_MIN
#      define ERTS_SINT64_MIN LONG_MIN
#    endif
# 404 "beam/sys.h"
#  elif SIZEOF_LONG_LONG == 8
# 405 "beam/sys.h"
#    define HAVE_INT64 1
typedef unsigned long long Uint64;
typedef long long          Sint64;
#    ifdef ULLONG_MAX
#      define ERTS_UINT64_MAX ULLONG_MAX
#    endif
# 411 "beam/sys.h"
#    ifdef LLONG_MAX
#      define ERTS_SINT64_MAX LLONG_MAX
#    endif
# 414 "beam/sys.h"
#    ifdef LLONG_MIN
#      define ERTS_SINT64_MIN LLONG_MIN
#    endif
# 417 "beam/sys.h"
#  else
# 418 "beam/sys.h"
#    error "No 64-bit integer type found"
#  endif
# 420 "beam/sys.h"
#endif
# 421 "beam/sys.h"

#ifndef ERTS_UINT64_MAX
#  define ERTS_UINT64_MAX (~((Uint64) 0))
#endif
# 425 "beam/sys.h"
#ifndef ERTS_SINT64_MAX
#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
#endif
# 428 "beam/sys.h"
#ifndef ERTS_SINT64_MIN
#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
#endif
# 431 "beam/sys.h"

#if SIZEOF_LONG == 4
typedef unsigned long Uint32;
typedef long          Sint32;
#elif SIZEOF_INT == 4
# 436 "beam/sys.h"
typedef unsigned int Uint32;
typedef int          Sint32;
#else
# 439 "beam/sys.h"
#error Found no appropriate type to use for 'Uint32' and 'Sint32'
#endif
# 441 "beam/sys.h"

#if SIZEOF_INT == 2
typedef unsigned int Uint16;
typedef int          Sint16;
#elif SIZEOF_SHORT == 2
# 446 "beam/sys.h"
typedef unsigned short Uint16;
typedef short          Sint16;
#else
# 449 "beam/sys.h"
#error Found no appropriate type to use for 'Uint16' and 'Sint16'
#endif
# 451 "beam/sys.h"

#if CHAR_BIT == 8
typedef unsigned char byte;
#else
# 455 "beam/sys.h"
#error Found no appropriate type to use for 'byte'
#endif
# 457 "beam/sys.h"

#if defined(ARCH_64) && !HAVE_INT64
#error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
#endif
# 461 "beam/sys.h"

#ifdef WORDS_BIGENDIAN
#  define ERTS_HUINT_HVAL_HIGH 0
#  define ERTS_HUINT_HVAL_LOW 1
#else
# 466 "beam/sys.h"
#  define ERTS_HUINT_HVAL_HIGH 1
#  define ERTS_HUINT_HVAL_LOW 0
#endif
# 469 "beam/sys.h"
#if ERTS_SIZEOF_TERM == 8
typedef union {
    Uint val;
    Uint32 hval[2];
} HUint;
#elif ERTS_SIZEOF_TERM == 4
# 475 "beam/sys.h"
typedef union {
    Uint val;
    Uint16 hval[2];
} HUint;
#else
# 480 "beam/sys.h"
#error "Unsupported size of term"
#endif
# 482 "beam/sys.h"

#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
    (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 486 "beam/sys.h"
# 487 "beam/sys.h"

/* needed by erl_smp.h */
int erts_send_warning_to_logger_str_nogl(char *);

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/sys.h"
# 492 "beam/sys.h"

#ifdef ERTS_WANT_BREAK_HANDLING
#  ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#    define ERTS_BREAK_REQUESTED \
  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
#  else
# 499 "beam/sys.h"
extern volatile int erts_break_requested;
#    define ERTS_BREAK_REQUESTED erts_break_requested
#  endif
# 502 "beam/sys.h"
void erts_do_break_handling(void);
#endif
# 504 "beam/sys.h"

#ifdef ERTS_WANT_GOT_SIGUSR1
#  ifndef UNIX
#    define ERTS_GOT_SIGUSR1 0
#  else
# 509 "beam/sys.h"
#    ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
#    else
# 513 "beam/sys.h"
extern volatile int erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
#    endif
# 516 "beam/sys.h"
#  endif
# 517 "beam/sys.h"
#endif
# 518 "beam/sys.h"

#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
extern erts_tsd_key_t erts_is_crash_dumping_key;
#define ERTS_SOMEONE_IS_CRASH_DUMPING \
  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
# 527 "beam/sys.h"
extern volatile int erts_writing_erl_crash_dump;
#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
# 531 "beam/sys.h"

/* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
   but be able to fall back on bcopy() etc on systems that don't have
   mem*(), but this doesn't work to well with memset()/bzero() - thus the
   memzero() macro.
*/

/* xxxP */
#if defined(USE_BCOPY)
#  define memcpy(a, b, c) bcopy((b), (a), (c))
#  define memcmp(a, b, c) bcmp((a), (b), (c))
#  define memzero(buf, len) bzero((buf), (len))
#else
# 544 "beam/sys.h"
#  define memzero(buf, len) memset((buf), '\0', (len))
#endif
# 546 "beam/sys.h"

/* Stuff that is useful for port programs, drivers, etc */

#ifdef ISC32			/* Too much for the Makefile... */
#  define signal	sigset
#  define NO_ASINH
#  define NO_ACOSH
#  define NO_ATANH
#  define NO_FTRUNCATE
#  define SIG_SIGHOLD
#  define _POSIX_SOURCE 
#  define _XOPEN_SOURCE
#endif
# 559 "beam/sys.h"

#ifdef QNX			/* Too much for the Makefile... */
#  define SYS_SELECT_H
#  define NO_ERF
#  define NO_ERFC
/* This definition doesn't take NaN into account, but matherr() gets those */
#  define isfinite(x) (fabs(x) != HUGE_VAL)
#  define USE_MATHERR
#  define HAVE_FINITE
#endif
# 569 "beam/sys.h"


#ifdef WANT_NONBLOCKING	    /* must define this to pull in fcntl.h/ioctl.h */

/* This is really a mess... We used to use fcntl O_NDELAY, but that seems
   to only work on SunOS 4 - in particular, on SysV-based systems
   (including Solaris 2), it does set non-blocking mode, but causes
   read() to return 0!!  fcntl O_NONBLOCK is specified by POSIX, and
   seems to work on most systems, with the notable exception of AIX,
   where the old ioctl FIONBIO is the *only* one that will set a *socket*
   in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
   pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */

#  ifdef __WIN32__

static unsigned long zero_value = 0, one_value = 1;
#    define SET_BLOCKING(fd)	{ if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); }
#    define SET_NONBLOCKING(fd)	ioctlsocket((fd), FIONBIO, &one_value)

#  else
# 589 "beam/sys.h"
#    ifdef NB_FIONBIO		/* Old BSD */
#if 0 /* expanded by -frewrite-includes */
#      include <sys/ioctl.h>
#endif /* expanded by -frewrite-includes */
# 590 "beam/sys.h"
# 591 "beam/sys.h"
  static const int zero_value = 0, one_value = 1;
#      define SET_BLOCKING(fd)         ioctl((fd), FIONBIO, &zero_value)
#      define SET_NONBLOCKING(fd)      ioctl((fd), FIONBIO, &one_value)
#      define ERRNO_BLOCK EWOULDBLOCK
#    else /* !NB_FIONBIO */
# 596 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#      include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 596 "beam/sys.h"
# 597 "beam/sys.h"
#      ifdef NB_O_NDELAY               /* Nothing needs this? */
#        define NB_FLAG O_NDELAY
#        ifndef ERRNO_BLOCK            /* allow override (e.g. EAGAIN) via Makefile */
#          define ERRNO_BLOCK EWOULDBLOCK
#        endif
# 602 "beam/sys.h"
#      else  /* !NB_O_NDELAY */	/* The True Way - POSIX!:-) */
# 603 "beam/sys.h"
#        define NB_FLAG O_NONBLOCK
#        define ERRNO_BLOCK EAGAIN
#      endif /* !NB_O_NDELAY */
# 606 "beam/sys.h"
#      define SET_BLOCKING(fd)         fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) & ~NB_FLAG)
#      define SET_NONBLOCKING(fd)      fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) | NB_FLAG)
#    endif /* !NB_FIONBIO */
# 611 "beam/sys.h"
#  endif /* !__WIN32__ */
# 612 "beam/sys.h"
#endif /* WANT_NONBLOCKING */
# 613 "beam/sys.h"

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);

/* Some special erts_exit() codes: */
#define ERTS_INTR_EXIT	-1		/* called from signal handler */
#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */

#define ERTS_INTERNAL_ERROR(What) \
    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
	     __FILE__, __LINE__, __func__, What)

Eterm erts_check_io_info(void *p);

/* Size of misc memory allocated from system dependent code */
Uint erts_sys_misc_mem_sz(void);

/* print stuff is declared here instead of in global.h, so sys stuff won't
   have to include global.h */
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 633 "beam/sys.h"
# 634 "beam/sys.h"

/* Io constants to erts_print and erts_putc */
#define ERTS_PRINT_STDERR	(2)
#define ERTS_PRINT_STDOUT	(1)
#define ERTS_PRINT_FILE		(-1)
#define ERTS_PRINT_SBUF		(-2)
#define ERTS_PRINT_SNBUF	(-3)
#define ERTS_PRINT_DSBUF	(-4)

#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF

typedef struct {
    char *buf;
    size_t size;
} erts_print_sn_buf;

int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
int erts_putc(int to, void *arg, char);			/* in utils.c */

/* logger stuff is declared here instead of in global.h, so sys files
   won't have to include global.h */

erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
int erts_send_info_to_logger_str(Eterm, char *); 
int erts_send_warning_to_logger_str(Eterm, char *);
int erts_send_error_to_logger_str(Eterm, char *);
int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
/* needed by erl_smp.h (declared above)
   int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);

typedef struct preload {
    char *name;			/* Name of module */
    int  size;			/* Size of code */
    unsigned char* code;	/* Code pointer */
} Preload;


/*
 * This structure contains options to all built in drivers.
 * None of the drivers use all of the fields.
 */

typedef struct _SysDriverOpts {
    Uint ifd;			/* Input file descriptor (fd driver). */
    Uint ofd;			/* Outputfile descriptor (fd driver). */
    int packet_bytes;		/* Number of bytes in packet header. */
    int read_write;		/* Read and write bits. */
    int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
    int redir_stderr;           /* Redirect stderr to stdout: TRUE/FALSE. */
    int hide_window;		/* Hide this windows (Windows). */
    int exit_status;		/* Report exit status of subprocess. */
    int overlapped_io;          /* Only has effect on windows NT et al */
    char *envir;		/* Environment of the port process, */
				/* in Windows format. */
    char **argv;                /* Argument vector in Unix'ish format. */
    char *wd;			/* Working directory. */
    unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
				   ERTS_SPAWN_EXTERNAL | both*/ 
    int parallelism;            /* Optimize for parallelism */
} SysDriverOpts;

extern char *erts_default_arg0;

extern char os_type[];

typedef struct {
    int have_os_monotonic_time;
    int have_corrected_os_monotonic_time;
    ErtsMonotonicTime os_monotonic_time_unit;
    ErtsMonotonicTime sys_clock_resolution;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
	int extended;
    } os_monotonic_time_info;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
    } os_system_time_info;
} ErtsSysInitTimeResult;

#define ERTS_SYS_INIT_TIME_RESULT_INITER \
    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}

extern void erts_init_sys_time_sup(void);
extern void sys_init_time(ErtsSysInitTimeResult *);
extern void erts_late_sys_init_time(void);
extern void erts_deliver_time(void);
extern void erts_time_remaining(SysTimeval *);
extern void erts_sys_init_float(void);
extern void erts_thread_init_float(void);
extern void erts_thread_disable_fpe(void);
ERTS_GLB_INLINE int erts_block_fpe(void);
ERTS_GLB_INLINE void erts_unblock_fpe(int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int erts_block_fpe(void)
{
    return erts_sys_block_fpe();
}

ERTS_GLB_INLINE void erts_unblock_fpe(int unmasked)
{
    erts_sys_unblock_fpe(unmasked);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 754 "beam/sys.h"


/* Dynamic library/driver loading */
typedef struct {
    char* str;
}ErtsSysDdllError;
#define ERTS_SYS_DDLL_ERROR_INIT {NULL}
extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*);
#define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL)
extern void *erts_sys_ddll_call_init(void *function);
extern void *erts_sys_ddll_call_nif_init(void *function);
extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*);
#define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL)
extern char *erts_sys_ddll_error(int code);



/*
 * System interfaces for startup.
 */
void erts_sys_schedule_interrupt(int set);
#ifdef ERTS_SMP
void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
void erts_sys_main_thread(void);
#endif
# 785 "beam/sys.h"

extern int erts_sys_prepare_crash_dump(int secs);
extern void erts_sys_pre_init(void);
extern void erl_sys_init(void);
extern void erl_sys_args(int *argc, char **argv);
extern void erl_sys_schedule(int);
void sys_tty_reset(int);

int sys_max_files(void);
void sys_init_io(void);
Preload* sys_preloaded(void);
unsigned char* sys_preload_begin(Preload*);
void sys_preload_end(Preload*);
int sys_get_key(int);
void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
		       UWord *ms_user_diff, UWord *ms_sys_diff);
void wall_clock_elapsed_time_both(UWord *ms_total, 
				  UWord *ms_diff);
void get_time(int *hour, int *minute, int *second);
void get_date(int *year, int *month, int *day);
void get_localtime(int *year, int *month, int *day, 
		   int *hour, int *minute, int *second);
void get_universaltime(int *year, int *month, int *day, 
		       int *hour, int *minute, int *second);
int seconds_to_univ(Sint64 seconds, 
		    Sint *year, Sint *month, Sint *day, 
		    Sint *hour, Sint *minute, Sint *second);
int univ_to_seconds(Sint year, Sint month, Sint day, 
		    Sint hour, Sint minute, Sint second,
		    Sint64* seconds);
int univ_to_local(
    Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second);
int local_to_univ(Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second, int isdst);
void get_now(Uint*, Uint*, Uint*);
struct ErtsSchedulerData_;
ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
ErtsMonotonicTime erts_get_time_offset(void);
void
erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
void get_sys_now(Uint*, Uint*, Uint*);
void set_break_quit(void (*)(void), void (*)(void));

void os_flavor(char*, unsigned);
void os_version(int*, int*, int*);
void init_getenv_state(GETENV_STATE *);
char * getenv_string(GETENV_STATE *);
void fini_getenv_state(GETENV_STATE *);

#define HAVE_ERTS_CHECK_IO_DEBUG
typedef struct {
    int no_used_fds;
    int no_driver_select_structs;
    int no_driver_event_structs;
} ErtsCheckIoDebugInfo;
int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);

int erts_sys_is_area_readable(char *start, char *stop);

/* xxxP */
#define SYS_DEFAULT_FLOAT_DECIMALS 20
void init_sys_float(void);
int sys_chars_to_double(char*, double*);
int sys_double_to_chars(double, char*, size_t);
int sys_double_to_chars_ext(double, char*, size_t, size_t);
int sys_double_to_chars_fast(double, char*, int, int, int);
void sys_get_pid(char *, size_t);

/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
int erts_sys_putenv(char *key, char *value);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_putenv_raw(char *key, char *value);
/* erts_sys_getenv() returns 0 on success (length of value string in
   *size), a value > 0 if value buffer is too small (*size is set to needed
   size), and a value < 0 on failure. */
int erts_sys_getenv(char *key, char *value, size_t *size);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_getenv_raw(char *key, char *value, size_t *size);
/* erts_sys_getenv__() is only allowed to be used in early init phase */
int erts_sys_getenv__(char *key, char *value, size_t *size);
/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
int erts_sys_unsetenv(char *key);

/* Easier to use, but not as efficient, environment functions */
char *erts_read_env(char *key);
void erts_free_read_env(void *value);

#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
extern void sys_thr_resume(erts_tid_t tid);
extern void sys_thr_suspend(erts_tid_t tid);
#endif
# 878 "beam/sys.h"

/* utils.c */

/* Options to sys_alloc_opt */
#define SYS_ALLOC_OPT_TRIM_THRESHOLD 0
#define SYS_ALLOC_OPT_TOP_PAD        1
#define SYS_ALLOC_OPT_MMAP_THRESHOLD 2
#define SYS_ALLOC_OPT_MMAP_MAX       3

/* Default values to sys_alloc_opt options */
#define ERTS_DEFAULT_TRIM_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_TOP_PAD         0
#define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_MMAP_MAX        64

int sys_alloc_opt(int, int);

typedef struct {
  int trim_threshold;
  int top_pad;
  int mmap_threshold;
  int mmap_max;
} SysAllocStat;

void sys_alloc_stat(SysAllocStat *);

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#undef ERTS_REFC_DEBUG
#define ERTS_REFC_DEBUG
#endif
# 908 "beam/sys.h"

typedef erts_smp_atomic_t erts_refc_t;

ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
				   erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
					   erts_aint_t min_val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}

ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 941 "beam/sys.h"
    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
# 943 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 955 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 968 "beam/sys.h"
    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
# 970 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 982 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
		 diff, val, min_val);
#else
# 995 "beam/sys.h"
    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
# 997 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 1009 "beam/sys.h"
    return val;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1013 "beam/sys.h"

#ifdef ERTS_ENABLE_KERNEL_POLL
extern int erts_use_kernel_poll;
#endif
# 1017 "beam/sys.h"

#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
#define sys_memset(s,c,n)    memset(s,c,n)
#define sys_memzero(s, n)    memset(s,'\0',n)
#define sys_strcmp(s1,s2)    strcmp(s1,s2)
#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
#define sys_strcpy(s1,s2)    strcpy(s1,s2)
#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
#define sys_strlen(s)        strlen(s)

/* define function symbols (needed in sys_drv_api) */
#define sys_fp_alloc     sys_alloc
#define sys_fp_realloc   sys_realloc
#define sys_fp_free      sys_free
#define sys_fp_memcpy    memcpy
#define sys_fp_memmove   memmove
#define sys_fp_memcmp    memcmp
#define sys_fp_memset    memset
/* #define sys_fp_memzero    elib_memzero */
#define sys_fp_strcmp    strcmp
#define sys_fp_strncmp   strncmp
#define sys_fp_strcpy    strcpy
#define sys_fp_strncpy   strncpy
#define sys_fp_strlen    strlen


/* Return codes from the nb_read and nb_write functions */
#define FD_READY 1
#define FD_CONTINUE 2
#define FD_ERROR 3



/* Standard set of integer macros  .. */

#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
                      (((Uint64)((unsigned char*) (s))[7])))

#define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                            ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
                            ((char*)(s))[2] = (char)((Sint64)(i) >> 40) & 0xff;\
                            ((char*)(s))[3] = (char)((Sint64)(i) >> 32) & 0xff;\
                            ((char*)(s))[4] = (char)((Sint64)(i) >> 24) & 0xff;\
                            ((char*)(s))[5] = (char)((Sint64)(i) >> 16) & 0xff;\
                            ((char*)(s))[6] = (char)((Sint64)(i) >> 8)  & 0xff;\
                            ((char*)(s))[7] = (char)((Sint64)(i))       & 0xff;\
                           } while (0) 

#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \
                      (((unsigned char*) (s))[1] << 16) | \
                      (((unsigned char*) (s))[2] << 8)  | \
                      (((unsigned char*) (s))[3]))

#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff;   \
                            ((char*)(s))[1] = (char)((i) >> 16) & 0xff;   \
                            ((char*)(s))[2] = (char)((i) >> 8)  & 0xff;   \
                            ((char*)(s))[3] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
                      (((unsigned char*) (s))[1] << 8)  | \
                      (((unsigned char*) (s))[2]))

#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                      (((unsigned char*)  (s))[1]))


#define put_int16(i, s) do {((char*)(s))[0] = (char)((i) >> 8) & 0xff;  \
                            ((char*)(s))[1] = (char)(i)        & 0xff;} \
                        while (0)

#define get_int8(s) ((((unsigned char*)  (s))[0] ))


#define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)


/*
 * Use DEBUGF as you would use printf, but use double parentheses:
 *
 *   DEBUGF(("Error: %s\n", error));
 *
 * The output will appear in a special console.
 */

#ifdef DEBUG
void erl_debug(char* format, ...);
void erl_bin_write(unsigned char *, int, int);

#  define DEBUGF(x) erl_debug x
#else
# 1121 "beam/sys.h"
#  define DEBUGF(x)
#endif
# 1123 "beam/sys.h"

#ifdef __WIN32__
#ifdef ARCH_64
#define ERTS_ALLOC_ALIGN_BYTES 16
#define ERTS_SMALL_ABS(Small) _abs64(Small) 
#else
# 1129 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1132 "beam/sys.h"
#else
# 1133 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1136 "beam/sys.h"

#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
#endif
# 1140 "beam/sys.h"

#ifdef __WIN32__
void call_break_handler(void);
char* last_error(void);
char* win32_errorstr(int);
#endif
# 1146 "beam/sys.h"

/************************************************************************
 * Find out the native filename encoding of the process (look at locale of 
 * Unix processes and just do UTF16 on windows 
 ************************************************************************/
#define ERL_FILENAME_UNKNOWN   (0)
#define ERL_FILENAME_LATIN1    (1)
#define ERL_FILENAME_UTF8      (2)
#define ERL_FILENAME_UTF8_MAC  (3)
#define ERL_FILENAME_WIN_WCHAR (4)

/************************************************************************
 * If a filename in for example list_dir is not in the right encoding, it
 * will be skipped in the resulting list, but depending on a startup setting
 * we will inform the user in different ways. These macros define the
 * different reactions to wrongly coded filenames. In the error case an
 * exception will be thrown by prim_file.
 ************************************************************************/
#define ERL_FILENAME_WARNING_WARNING (0)
#define ERL_FILENAME_WARNING_IGNORE (1)
#define ERL_FILENAME_WARNING_ERROR (2)

/***********************************************************************
 * The user can request a range of character that he/she consider
 * printable. Currently this can be either latin1 or unicode, but
 * in the future a set of ranges, or languages, could be specified.
 ***********************************************************************/
#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0)
#define ERL_PRINTABLE_CHARACTERS_UNICODE (1)

int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
void erts_set_user_requested_filename_encoding(int encoding, int warning);
int erts_get_user_requested_filename_encoding(void);
int erts_get_filename_warning_type(void);
/* This function is called from erl_init. The setting is read by BIF's 
   in io/io_lib. Setting is not atomic. */
void erts_set_printable_characters(int range);
/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */
int erts_get_printable_characters(void);

void erts_init_sys_common_misc(void);

#endif
# 1190 "beam/sys.h"
# 33 "beam/erl_lock_check.h" 2

#ifndef ERTS_LOCK_CHECK_H__
#define ERTS_LOCK_CHECK_H__

#ifdef ERTS_ENABLE_LOCK_CHECK

#ifndef ERTS_ENABLE_LOCK_POSITION
/* Enable in order for _x variants of mtx functions to be used. */
#define ERTS_ENABLE_LOCK_POSITION 1
#endif
# 43 "beam/erl_lock_check.h"

typedef struct {
    int inited;
    Sint16 id;
    Uint16 flags;
    UWord extra;
} erts_lc_lock_t;

#define ERTS_LC_INITITALIZED 0x7f7f7f7f


#define ERTS_LC_FLG_LT_SPINLOCK		(((Uint16) 1) << 0)
#define ERTS_LC_FLG_LT_RWSPINLOCK	(((Uint16) 1) << 1)
#define ERTS_LC_FLG_LT_MUTEX		(((Uint16) 1) << 2)
#define ERTS_LC_FLG_LT_RWMUTEX		(((Uint16) 1) << 3)
#define ERTS_LC_FLG_LT_PROCLOCK		(((Uint16) 1) << 4)

#define ERTS_LC_FLG_LO_READ		(((Uint16) 1) << 5)
#define ERTS_LC_FLG_LO_WRITE		(((Uint16) 1) << 6)

#define ERTS_LC_FLG_LO_READ_WRITE	(ERTS_LC_FLG_LO_READ		\
					 | ERTS_LC_FLG_LO_WRITE)

#define ERTS_LC_FLG_LT_ALL		(ERTS_LC_FLG_LT_SPINLOCK	\
					 | ERTS_LC_FLG_LT_RWSPINLOCK	\
					 | ERTS_LC_FLG_LT_MUTEX		\
					 | ERTS_LC_FLG_LT_RWMUTEX	\
					 | ERTS_LC_FLG_LT_PROCLOCK)

#define ERTS_LC_FLG_LO_ALL		(ERTS_LC_FLG_LO_READ		\
					 | ERTS_LC_FLG_LO_WRITE)


#define ERTS_LC_LOCK_INIT(ID, X, F) {ERTS_LC_INITITALIZED, (ID), (F), (X)}

void erts_lc_init(void);
void erts_lc_late_init(void);
Sint16 erts_lc_get_lock_order_id(char *name);
void erts_lc_check(erts_lc_lock_t *have, int have_len,
		   erts_lc_lock_t *have_not, int have_not_len);
void erts_lc_check_exact(erts_lc_lock_t *have, int have_len);
void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len);
void erts_lc_have_lock_ids(int *resv, int *ids, int len);
void erts_lc_check_no_locked_of_type(Uint16 flags);
int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags);
void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, Uint16 op_flags,
			   char *file, unsigned int line);
void erts_lc_lock_flg_x(erts_lc_lock_t *lck, Uint16 op_flags,
			char *file, unsigned int line);
void erts_lc_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, Uint16 op_flags);
int erts_lc_trylock_force_busy(erts_lc_lock_t *lck);
void erts_lc_trylock_x(int locked, erts_lc_lock_t *lck,
		     char* file, unsigned int line);
void erts_lc_lock_x(erts_lc_lock_t *lck, char* file, unsigned int line);
void erts_lc_unlock(erts_lc_lock_t *lck);
void erts_lc_might_unlock(erts_lc_lock_t *lck);
void erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags);
void erts_lc_init_lock_x(erts_lc_lock_t *lck, char *name, Uint16 flags, Eterm extra);
void erts_lc_destroy_lock(erts_lc_lock_t *lck);
void erts_lc_fail(char *fmt, ...);
int erts_lc_assert_failed(char *file, int line, char *assertion);
void erts_lc_set_thread_name(char *thread_name);
void erts_lc_pll(void);

void erts_lc_require_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags,
			      char *file, unsigned int line);
void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags);

void erts_lc_require_lock(erts_lc_lock_t *lck, char *file, unsigned int line);
void erts_lc_unrequire_lock(erts_lc_lock_t *lck);

int erts_lc_is_emu_thr(void);

#define ERTS_LC_ASSERT(A) \
    ((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A)))
#ifdef ERTS_SMP
#define ERTS_SMP_LC_ASSERT(A) ERTS_LC_ASSERT(A)
#else
# 122 "beam/erl_lock_check.h"
#define ERTS_SMP_LC_ASSERT(A) ((void) 1)
#endif
# 124 "beam/erl_lock_check.h"
#else /* #ifdef ERTS_ENABLE_LOCK_CHECK */
# 125 "beam/erl_lock_check.h"
#define ERTS_SMP_LC_ASSERT(A) ((void) 1)
#define ERTS_LC_ASSERT(A) ((void) 1)
#endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
# 128 "beam/erl_lock_check.h"

#define erts_lc_lock(lck) erts_lc_lock_x(lck,__FILE__,__LINE__)
#define erts_lc_trylock(res,lck) erts_lc_trylock_x(res,lck,__FILE__,__LINE__)
#define erts_lc_lock_flg(lck) erts_lc_lock_flg_x(lck,__FILE__,__LINE__)
#define erts_lc_trylock_flg(res,lck) erts_lc_trylock_flg_x(res,lck,__FILE__,__LINE__)

#endif /* #ifndef ERTS_LOCK_CHECK_H__ */
# 135 "beam/erl_lock_check.h"
# 267 "beam/erl_threads.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_count.h"
#endif /* expanded by -frewrite-includes */
# 267 "beam/erl_threads.h"
# 1 "beam/erl_lock_count.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2008-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description:	Statistics for locks.
 *
 * Author:	Björn-Egil Dahlberg
 * Date:	2008-07-03
 * Abstract:
 * 	Locks statistics internal representation.
 * 	
 * 	Conceptual representation, 
 * 	- set name
 * 	| - id (the unique lock)
 * 	| | - lock type
 * 	| | - statistics
 * 	| | | - location (file and line number)
 * 	| | | - tries
 * 	| | | - collisions (including trylock busy)
 * 	| | | - timer (time spent in waiting for lock)
 * 	| | | - n_timer (collisions excluding trylock busy)
 * 	| | | - histogram
 * 	| | | | - # 0 = log2(lock wait_time ns)
 * 	| | | | - ...
 * 	| | | | - # n = log2(lock wait_time ns)
 *
 * 	Each instance of a lock is the unique lock, i.e. set and id in that set.
 * 	For each lock there is a set of statistics with where and what impact
 * 	the lock aqusition had.
 *
 * 	Runtime options
 * 	- suspend, used when internal lock-counting can't be applied. For instance
 * 	  when allocating a term for the outside and halloc needs to be used.
 * 	  Default: off.
 * 	- location, reserved and not used.
 * 	- proclock, disable proclock counting. Used when performance might be an
 * 	  issue. Accessible from erts_debug:lock_counters({process_locks, bool()}).
 * 	  Default: off.
 * 	- copysave, enable saving of destroyed locks (and thereby its statistics).
 * 	  If memory constraints is an issue this need to be disabled.
 * 	  Accessible from erts_debug:lock_counters({copy_save, bool()}).
 * 	  Default: off.
 *
 */

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 63 "beam/erl_lock_count.h"
# 1 "beam/sys.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __SYS_H__
#define __SYS_H__

#ifdef ERTS_INLINE
#  ifndef ERTS_CAN_INLINE
#    define ERTS_CAN_INLINE 1
#  endif
# 28 "beam/sys.h"
#else
# 29 "beam/sys.h"
#  if defined(__GNUC__)
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline__
#  elif defined(__WIN32__)
# 33 "beam/sys.h"
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline
#  else
# 36 "beam/sys.h"
#    define ERTS_CAN_INLINE 0
#    define ERTS_INLINE
#  endif
# 39 "beam/sys.h"
#endif
# 40 "beam/sys.h"

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#  undef ERTS_CAN_INLINE
#  define ERTS_CAN_INLINE 0
#  undef ERTS_INLINE
#  define ERTS_INLINE
#endif
# 47 "beam/sys.h"

#if ERTS_CAN_INLINE
#define ERTS_GLB_INLINE static ERTS_INLINE
#else
# 51 "beam/sys.h"
#define ERTS_GLB_INLINE
#endif
# 53 "beam/sys.h"

#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
#else
# 57 "beam/sys.h"
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
# 59 "beam/sys.h"

#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
#  define NO_FPE_SIGNALS
#endif
# 63 "beam/sys.h"

#ifdef DISABLE_CHILD_WAITER_THREAD
#undef ENABLE_CHILD_WAITER_THREAD
#endif
# 67 "beam/sys.h"

#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
#undef ENABLE_CHILD_WAITER_THREAD
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
# 72 "beam/sys.h"

#define ERTS_I64_LITERAL(X) X##LL

#if defined (__WIN32__)
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_sys.h"
#endif /* expanded by -frewrite-includes */
# 76 "beam/sys.h"
# 77 "beam/sys.h"
#elif defined (__OSE__)
# 78 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_ose_sys.h"
#endif /* expanded by -frewrite-includes */
# 78 "beam/sys.h"
# 79 "beam/sys.h"
#else 
# 80 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_unix_sys.h"
#endif /* expanded by -frewrite-includes */
# 80 "beam/sys.h"
# 81 "beam/sys.h"
#ifndef UNIX
#  define UNIX 1
#endif
# 84 "beam/sys.h"
#endif
# 85 "beam/sys.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_misc_utils.h"
#endif /* expanded by -frewrite-includes */
# 86 "beam/sys.h"
# 87 "beam/sys.h"

/*
 * To allow building of Universal Binaries for Mac OS X,
 * we must not depend on the endian detected by the configure script.
 */
#if defined(__APPLE__)
#  if defined(__BIG_ENDIAN__) && !defined(WORDS_BIGENDIAN)
#    define WORDS_BIGENDIAN 1
#  elif !defined(__BIG_ENDIAN__) && defined(WORDS_BIGENDIAN)
# 96 "beam/sys.h"
#    undef WORDS_BIGENDIAN
#  endif
# 98 "beam/sys.h"
#endif
# 99 "beam/sys.h"

/*
 * Make sure we have a type for FD's (used by erl_check_io)
 */

#ifndef ERTS_SYS_FD_TYPE
#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
typedef int ErtsSysFdType;
#else
# 108 "beam/sys.h"
#ifndef ERTS_SYS_FD_INVALID
# error missing ERTS_SYS_FD_INVALID
#endif
# 111 "beam/sys.h"
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
# 113 "beam/sys.h"

#if !defined(__GNUC__)
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 117 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 120 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 123 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 126 "beam/sys.h"

#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
#  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
#  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# 131 "beam/sys.h"
#  define ERTS_LIKELY(BOOL)   (BOOL)
#  define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
# 134 "beam/sys.h"
#ifdef __GNUC__
#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
#  else
# 138 "beam/sys.h"
#    define ERTS_DECLARE_DUMMY(X) X
#  endif
# 140 "beam/sys.h"
#else
# 141 "beam/sys.h"
#  define ERTS_DECLARE_DUMMY(X) X
#endif
# 143 "beam/sys.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 149 "beam/sys.h"
#      define __func__ __FUNCTION__
#    endif
# 151 "beam/sys.h"
#  endif
# 152 "beam/sys.h"
#endif
# 153 "beam/sys.h"

#define ERTS_MK_VSN_INT(Major, Minor, Build) \
    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))

#ifndef ERTS_EXIT_AFTER_DUMP
#  define ERTS_EXIT_AFTER_DUMP exit
#endif
# 160 "beam/sys.h"

/* In VC++, noreturn is a declspec that has to be before the types,
 * but in GNUC it is an att ribute to be placed between return type
 * and function name, hence __decl_noreturn <types> __noreturn <function name>
 *
 * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
 */
#if __GNUC__
#  define __decl_noreturn
#  ifndef __noreturn
#     define __noreturn __attribute__((noreturn))
#  endif
# 172 "beam/sys.h"
#else
# 173 "beam/sys.h"
#  if defined(__WIN32__) && defined(_MSC_VER)
#    define __noreturn
#    define __decl_noreturn __declspec(noreturn)
#  else
# 177 "beam/sys.h"
#    define __noreturn
#    define __decl_noreturn
#  endif
# 180 "beam/sys.h"
#endif
# 181 "beam/sys.h"

#define ERTS_ASSERT(e) \
    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))

__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
						 const char* file, int line);

#ifdef DEBUG
#  define ASSERT(e) ERTS_ASSERT(e)
#else
# 191 "beam/sys.h"
#  define ASSERT(e) ((void) 1)
#endif
# 193 "beam/sys.h"

/* ERTS_UNDEF can be used to silence false warnings about
 * "variable may be used uninitialized" while keeping the variable
 * marked as undefined by valgrind.
 */
#ifdef VALGRIND
#  define ERTS_UNDEF(V,I)
#else
# 201 "beam/sys.h"
#  define ERTS_UNDEF(V,I) V = I
#endif
# 203 "beam/sys.h"

/*
 * Compile time assert
 * (the actual compiler error msg can be a bit confusing)
 */
#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
# define ERTS_CT_ASSERT(e) \
    do { \
	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
    } while(0)
#else
# 214 "beam/sys.h"
# define ERTS_CT_ASSERT(e) \
    do { \
        enum { compile_time_assert__ = 1/(e) }; \
    } while (0)
#endif
# 219 "beam/sys.h"

/*
 * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
 * But MSC doesn't define __STDC__, unless we compile with the -Za
 * flag (strict ANSI C, no Microsoft extension).  Compiling with -Za
 * doesn't work: some Microsoft headers fail to compile...
 *
 * Solution: Test if __STDC__ or _MSC_VER is defined.
 *
 * Note: Simply defining __STDC__ doesn't work, as some Microsoft
 * headers will fail to compile!
 */

#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 232 "beam/sys.h"
# 233 "beam/sys.h"

/* This isn't sys-dependent, but putting it here benefits sys.c and drivers
   - allow use of 'const' regardless of compiler */

#if !defined(__STDC__) && !defined(_MSC_VER)
#  define const
#endif
# 240 "beam/sys.h"

#undef __deprecated
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
#  define __deprecated __attribute__((deprecated))
#else
# 245 "beam/sys.h"
#  define __deprecated
#endif
# 247 "beam/sys.h"
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
#  define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
#else
# 250 "beam/sys.h"
#  define erts_align_attribute(SZ)
#endif
# 252 "beam/sys.h"

/*
** Data types:
**
** Eterm: A tagged erlang term (possibly 64 bits)
** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
** UInt:  An unsigned integer exactly as large as an Eterm.
** SInt:  A signed integer exactly as large as an eterm and therefor large
**        enough to hold the return value of the signed_val() macro.
** UWord: An unsigned integer at least as large as a void * and also as large
**          or larger than an Eterm
** SWord: A signed integer at least as large as a void * and also as large
**          or larger than an Eterm
** Uint32: An unsigned integer of 32 bits exactly
** Sint32: A signed integer of 32 bits exactly
** Uint16: An unsigned integer of 16 bits exactly
** Sint16: A signed integer of 16 bits exactly.
*/

#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
#error Cannot handle this combination of int/long/void*/size_t sizes
#endif
# 276 "beam/sys.h"

#if SIZEOF_VOID_P == 8
#undef  ARCH_32
#define ARCH_64
#define ERTS_SIZEOF_TERM 8
#elif SIZEOF_VOID_P == 4
# 282 "beam/sys.h"
#define ARCH_32
#undef  ARCH_64
#define ERTS_SIZEOF_TERM 4
#else
# 286 "beam/sys.h"
#error Neither 32 nor 64 bit architecture
#endif
# 288 "beam/sys.h"
#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
#    define HALFWORD_HEAP 1
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND) ASSERT(COND)
#    undef ERTS_SIZEOF_TERM
#    define ERTS_SIZEOF_TERM 4
#else
# 295 "beam/sys.h"
#    define HALFWORD_HEAP 0
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND)
#endif
# 299 "beam/sys.h"

#if SIZEOF_VOID_P != SIZEOF_SIZE_T
#error sizeof(void*) != sizeof(size_t)
#endif
# 303 "beam/sys.h"

#if HALFWORD_HEAP

#if SIZEOF_INT == 4
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define ERTS_UINT_MAX UINT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#else
# 314 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 316 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long UWord;
typedef long          SWord;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#elif SIZEOF_VOID_P == SIZEOF_INT
# 325 "beam/sys.h"
typedef unsigned int UWord;
typedef int          SWord;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 332 "beam/sys.h"
typedef unsigned long long UWord;
typedef long long          SWord;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#else
# 339 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 341 "beam/sys.h"

#else /* !HALFWORD_HEAP */
# 343 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long Eterm;
typedef unsigned long Uint;
typedef long          Sint;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_INT
# 355 "beam/sys.h"
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 365 "beam/sys.h"
typedef unsigned long long Eterm;
typedef unsigned long long Uint;
typedef long long          Sint;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
#if defined(__WIN32__)
#define ErtsStrToSint _strtoi64
#else
# 376 "beam/sys.h"
#define ErtsStrToSint strtoll
#endif
# 378 "beam/sys.h"
#else
# 379 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 381 "beam/sys.h"

typedef Uint UWord;
typedef Sint SWord;
#define ERTS_UINT_MAX ERTS_UWORD_MAX

#endif /* HALFWORD_HEAP */
# 387 "beam/sys.h"

typedef UWord BeamInstr;

#ifndef HAVE_INT64
#  if SIZEOF_LONG == 8
#    define HAVE_INT64 1
typedef unsigned long Uint64;
typedef long          Sint64;
#    ifdef ULONG_MAX
#      define ERTS_UINT64_MAX ULONG_MAX
#    endif
# 398 "beam/sys.h"
#    ifdef LONG_MAX
#      define ERTS_SINT64_MAX LONG_MAX
#    endif
# 401 "beam/sys.h"
#    ifdef LONG_MIN
#      define ERTS_SINT64_MIN LONG_MIN
#    endif
# 404 "beam/sys.h"
#  elif SIZEOF_LONG_LONG == 8
# 405 "beam/sys.h"
#    define HAVE_INT64 1
typedef unsigned long long Uint64;
typedef long long          Sint64;
#    ifdef ULLONG_MAX
#      define ERTS_UINT64_MAX ULLONG_MAX
#    endif
# 411 "beam/sys.h"
#    ifdef LLONG_MAX
#      define ERTS_SINT64_MAX LLONG_MAX
#    endif
# 414 "beam/sys.h"
#    ifdef LLONG_MIN
#      define ERTS_SINT64_MIN LLONG_MIN
#    endif
# 417 "beam/sys.h"
#  else
# 418 "beam/sys.h"
#    error "No 64-bit integer type found"
#  endif
# 420 "beam/sys.h"
#endif
# 421 "beam/sys.h"

#ifndef ERTS_UINT64_MAX
#  define ERTS_UINT64_MAX (~((Uint64) 0))
#endif
# 425 "beam/sys.h"
#ifndef ERTS_SINT64_MAX
#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
#endif
# 428 "beam/sys.h"
#ifndef ERTS_SINT64_MIN
#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
#endif
# 431 "beam/sys.h"

#if SIZEOF_LONG == 4
typedef unsigned long Uint32;
typedef long          Sint32;
#elif SIZEOF_INT == 4
# 436 "beam/sys.h"
typedef unsigned int Uint32;
typedef int          Sint32;
#else
# 439 "beam/sys.h"
#error Found no appropriate type to use for 'Uint32' and 'Sint32'
#endif
# 441 "beam/sys.h"

#if SIZEOF_INT == 2
typedef unsigned int Uint16;
typedef int          Sint16;
#elif SIZEOF_SHORT == 2
# 446 "beam/sys.h"
typedef unsigned short Uint16;
typedef short          Sint16;
#else
# 449 "beam/sys.h"
#error Found no appropriate type to use for 'Uint16' and 'Sint16'
#endif
# 451 "beam/sys.h"

#if CHAR_BIT == 8
typedef unsigned char byte;
#else
# 455 "beam/sys.h"
#error Found no appropriate type to use for 'byte'
#endif
# 457 "beam/sys.h"

#if defined(ARCH_64) && !HAVE_INT64
#error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
#endif
# 461 "beam/sys.h"

#ifdef WORDS_BIGENDIAN
#  define ERTS_HUINT_HVAL_HIGH 0
#  define ERTS_HUINT_HVAL_LOW 1
#else
# 466 "beam/sys.h"
#  define ERTS_HUINT_HVAL_HIGH 1
#  define ERTS_HUINT_HVAL_LOW 0
#endif
# 469 "beam/sys.h"
#if ERTS_SIZEOF_TERM == 8
typedef union {
    Uint val;
    Uint32 hval[2];
} HUint;
#elif ERTS_SIZEOF_TERM == 4
# 475 "beam/sys.h"
typedef union {
    Uint val;
    Uint16 hval[2];
} HUint;
#else
# 480 "beam/sys.h"
#error "Unsupported size of term"
#endif
# 482 "beam/sys.h"

#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
    (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 486 "beam/sys.h"
# 487 "beam/sys.h"

/* needed by erl_smp.h */
int erts_send_warning_to_logger_str_nogl(char *);

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/sys.h"
# 492 "beam/sys.h"

#ifdef ERTS_WANT_BREAK_HANDLING
#  ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#    define ERTS_BREAK_REQUESTED \
  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
#  else
# 499 "beam/sys.h"
extern volatile int erts_break_requested;
#    define ERTS_BREAK_REQUESTED erts_break_requested
#  endif
# 502 "beam/sys.h"
void erts_do_break_handling(void);
#endif
# 504 "beam/sys.h"

#ifdef ERTS_WANT_GOT_SIGUSR1
#  ifndef UNIX
#    define ERTS_GOT_SIGUSR1 0
#  else
# 509 "beam/sys.h"
#    ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
#    else
# 513 "beam/sys.h"
extern volatile int erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
#    endif
# 516 "beam/sys.h"
#  endif
# 517 "beam/sys.h"
#endif
# 518 "beam/sys.h"

#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
extern erts_tsd_key_t erts_is_crash_dumping_key;
#define ERTS_SOMEONE_IS_CRASH_DUMPING \
  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
# 527 "beam/sys.h"
extern volatile int erts_writing_erl_crash_dump;
#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
# 531 "beam/sys.h"

/* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
   but be able to fall back on bcopy() etc on systems that don't have
   mem*(), but this doesn't work to well with memset()/bzero() - thus the
   memzero() macro.
*/

/* xxxP */
#if defined(USE_BCOPY)
#  define memcpy(a, b, c) bcopy((b), (a), (c))
#  define memcmp(a, b, c) bcmp((a), (b), (c))
#  define memzero(buf, len) bzero((buf), (len))
#else
# 544 "beam/sys.h"
#  define memzero(buf, len) memset((buf), '\0', (len))
#endif
# 546 "beam/sys.h"

/* Stuff that is useful for port programs, drivers, etc */

#ifdef ISC32			/* Too much for the Makefile... */
#  define signal	sigset
#  define NO_ASINH
#  define NO_ACOSH
#  define NO_ATANH
#  define NO_FTRUNCATE
#  define SIG_SIGHOLD
#  define _POSIX_SOURCE 
#  define _XOPEN_SOURCE
#endif
# 559 "beam/sys.h"

#ifdef QNX			/* Too much for the Makefile... */
#  define SYS_SELECT_H
#  define NO_ERF
#  define NO_ERFC
/* This definition doesn't take NaN into account, but matherr() gets those */
#  define isfinite(x) (fabs(x) != HUGE_VAL)
#  define USE_MATHERR
#  define HAVE_FINITE
#endif
# 569 "beam/sys.h"


#ifdef WANT_NONBLOCKING	    /* must define this to pull in fcntl.h/ioctl.h */

/* This is really a mess... We used to use fcntl O_NDELAY, but that seems
   to only work on SunOS 4 - in particular, on SysV-based systems
   (including Solaris 2), it does set non-blocking mode, but causes
   read() to return 0!!  fcntl O_NONBLOCK is specified by POSIX, and
   seems to work on most systems, with the notable exception of AIX,
   where the old ioctl FIONBIO is the *only* one that will set a *socket*
   in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
   pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */

#  ifdef __WIN32__

static unsigned long zero_value = 0, one_value = 1;
#    define SET_BLOCKING(fd)	{ if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); }
#    define SET_NONBLOCKING(fd)	ioctlsocket((fd), FIONBIO, &one_value)

#  else
# 589 "beam/sys.h"
#    ifdef NB_FIONBIO		/* Old BSD */
#if 0 /* expanded by -frewrite-includes */
#      include <sys/ioctl.h>
#endif /* expanded by -frewrite-includes */
# 590 "beam/sys.h"
# 591 "beam/sys.h"
  static const int zero_value = 0, one_value = 1;
#      define SET_BLOCKING(fd)         ioctl((fd), FIONBIO, &zero_value)
#      define SET_NONBLOCKING(fd)      ioctl((fd), FIONBIO, &one_value)
#      define ERRNO_BLOCK EWOULDBLOCK
#    else /* !NB_FIONBIO */
# 596 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#      include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 596 "beam/sys.h"
# 597 "beam/sys.h"
#      ifdef NB_O_NDELAY               /* Nothing needs this? */
#        define NB_FLAG O_NDELAY
#        ifndef ERRNO_BLOCK            /* allow override (e.g. EAGAIN) via Makefile */
#          define ERRNO_BLOCK EWOULDBLOCK
#        endif
# 602 "beam/sys.h"
#      else  /* !NB_O_NDELAY */	/* The True Way - POSIX!:-) */
# 603 "beam/sys.h"
#        define NB_FLAG O_NONBLOCK
#        define ERRNO_BLOCK EAGAIN
#      endif /* !NB_O_NDELAY */
# 606 "beam/sys.h"
#      define SET_BLOCKING(fd)         fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) & ~NB_FLAG)
#      define SET_NONBLOCKING(fd)      fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) | NB_FLAG)
#    endif /* !NB_FIONBIO */
# 611 "beam/sys.h"
#  endif /* !__WIN32__ */
# 612 "beam/sys.h"
#endif /* WANT_NONBLOCKING */
# 613 "beam/sys.h"

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);

/* Some special erts_exit() codes: */
#define ERTS_INTR_EXIT	-1		/* called from signal handler */
#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */

#define ERTS_INTERNAL_ERROR(What) \
    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
	     __FILE__, __LINE__, __func__, What)

Eterm erts_check_io_info(void *p);

/* Size of misc memory allocated from system dependent code */
Uint erts_sys_misc_mem_sz(void);

/* print stuff is declared here instead of in global.h, so sys stuff won't
   have to include global.h */
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 633 "beam/sys.h"
# 634 "beam/sys.h"

/* Io constants to erts_print and erts_putc */
#define ERTS_PRINT_STDERR	(2)
#define ERTS_PRINT_STDOUT	(1)
#define ERTS_PRINT_FILE		(-1)
#define ERTS_PRINT_SBUF		(-2)
#define ERTS_PRINT_SNBUF	(-3)
#define ERTS_PRINT_DSBUF	(-4)

#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF

typedef struct {
    char *buf;
    size_t size;
} erts_print_sn_buf;

int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
int erts_putc(int to, void *arg, char);			/* in utils.c */

/* logger stuff is declared here instead of in global.h, so sys files
   won't have to include global.h */

erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
int erts_send_info_to_logger_str(Eterm, char *); 
int erts_send_warning_to_logger_str(Eterm, char *);
int erts_send_error_to_logger_str(Eterm, char *);
int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
/* needed by erl_smp.h (declared above)
   int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);

typedef struct preload {
    char *name;			/* Name of module */
    int  size;			/* Size of code */
    unsigned char* code;	/* Code pointer */
} Preload;


/*
 * This structure contains options to all built in drivers.
 * None of the drivers use all of the fields.
 */

typedef struct _SysDriverOpts {
    Uint ifd;			/* Input file descriptor (fd driver). */
    Uint ofd;			/* Outputfile descriptor (fd driver). */
    int packet_bytes;		/* Number of bytes in packet header. */
    int read_write;		/* Read and write bits. */
    int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
    int redir_stderr;           /* Redirect stderr to stdout: TRUE/FALSE. */
    int hide_window;		/* Hide this windows (Windows). */
    int exit_status;		/* Report exit status of subprocess. */
    int overlapped_io;          /* Only has effect on windows NT et al */
    char *envir;		/* Environment of the port process, */
				/* in Windows format. */
    char **argv;                /* Argument vector in Unix'ish format. */
    char *wd;			/* Working directory. */
    unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
				   ERTS_SPAWN_EXTERNAL | both*/ 
    int parallelism;            /* Optimize for parallelism */
} SysDriverOpts;

extern char *erts_default_arg0;

extern char os_type[];

typedef struct {
    int have_os_monotonic_time;
    int have_corrected_os_monotonic_time;
    ErtsMonotonicTime os_monotonic_time_unit;
    ErtsMonotonicTime sys_clock_resolution;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
	int extended;
    } os_monotonic_time_info;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
    } os_system_time_info;
} ErtsSysInitTimeResult;

#define ERTS_SYS_INIT_TIME_RESULT_INITER \
    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}

extern void erts_init_sys_time_sup(void);
extern void sys_init_time(ErtsSysInitTimeResult *);
extern void erts_late_sys_init_time(void);
extern void erts_deliver_time(void);
extern void erts_time_remaining(SysTimeval *);
extern void erts_sys_init_float(void);
extern void erts_thread_init_float(void);
extern void erts_thread_disable_fpe(void);
ERTS_GLB_INLINE int erts_block_fpe(void);
ERTS_GLB_INLINE void erts_unblock_fpe(int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int erts_block_fpe(void)
{
    return erts_sys_block_fpe();
}

ERTS_GLB_INLINE void erts_unblock_fpe(int unmasked)
{
    erts_sys_unblock_fpe(unmasked);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 754 "beam/sys.h"


/* Dynamic library/driver loading */
typedef struct {
    char* str;
}ErtsSysDdllError;
#define ERTS_SYS_DDLL_ERROR_INIT {NULL}
extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*);
#define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL)
extern void *erts_sys_ddll_call_init(void *function);
extern void *erts_sys_ddll_call_nif_init(void *function);
extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*);
#define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL)
extern char *erts_sys_ddll_error(int code);



/*
 * System interfaces for startup.
 */
void erts_sys_schedule_interrupt(int set);
#ifdef ERTS_SMP
void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
void erts_sys_main_thread(void);
#endif
# 785 "beam/sys.h"

extern int erts_sys_prepare_crash_dump(int secs);
extern void erts_sys_pre_init(void);
extern void erl_sys_init(void);
extern void erl_sys_args(int *argc, char **argv);
extern void erl_sys_schedule(int);
void sys_tty_reset(int);

int sys_max_files(void);
void sys_init_io(void);
Preload* sys_preloaded(void);
unsigned char* sys_preload_begin(Preload*);
void sys_preload_end(Preload*);
int sys_get_key(int);
void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
		       UWord *ms_user_diff, UWord *ms_sys_diff);
void wall_clock_elapsed_time_both(UWord *ms_total, 
				  UWord *ms_diff);
void get_time(int *hour, int *minute, int *second);
void get_date(int *year, int *month, int *day);
void get_localtime(int *year, int *month, int *day, 
		   int *hour, int *minute, int *second);
void get_universaltime(int *year, int *month, int *day, 
		       int *hour, int *minute, int *second);
int seconds_to_univ(Sint64 seconds, 
		    Sint *year, Sint *month, Sint *day, 
		    Sint *hour, Sint *minute, Sint *second);
int univ_to_seconds(Sint year, Sint month, Sint day, 
		    Sint hour, Sint minute, Sint second,
		    Sint64* seconds);
int univ_to_local(
    Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second);
int local_to_univ(Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second, int isdst);
void get_now(Uint*, Uint*, Uint*);
struct ErtsSchedulerData_;
ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
ErtsMonotonicTime erts_get_time_offset(void);
void
erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
void get_sys_now(Uint*, Uint*, Uint*);
void set_break_quit(void (*)(void), void (*)(void));

void os_flavor(char*, unsigned);
void os_version(int*, int*, int*);
void init_getenv_state(GETENV_STATE *);
char * getenv_string(GETENV_STATE *);
void fini_getenv_state(GETENV_STATE *);

#define HAVE_ERTS_CHECK_IO_DEBUG
typedef struct {
    int no_used_fds;
    int no_driver_select_structs;
    int no_driver_event_structs;
} ErtsCheckIoDebugInfo;
int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);

int erts_sys_is_area_readable(char *start, char *stop);

/* xxxP */
#define SYS_DEFAULT_FLOAT_DECIMALS 20
void init_sys_float(void);
int sys_chars_to_double(char*, double*);
int sys_double_to_chars(double, char*, size_t);
int sys_double_to_chars_ext(double, char*, size_t, size_t);
int sys_double_to_chars_fast(double, char*, int, int, int);
void sys_get_pid(char *, size_t);

/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
int erts_sys_putenv(char *key, char *value);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_putenv_raw(char *key, char *value);
/* erts_sys_getenv() returns 0 on success (length of value string in
   *size), a value > 0 if value buffer is too small (*size is set to needed
   size), and a value < 0 on failure. */
int erts_sys_getenv(char *key, char *value, size_t *size);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_getenv_raw(char *key, char *value, size_t *size);
/* erts_sys_getenv__() is only allowed to be used in early init phase */
int erts_sys_getenv__(char *key, char *value, size_t *size);
/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
int erts_sys_unsetenv(char *key);

/* Easier to use, but not as efficient, environment functions */
char *erts_read_env(char *key);
void erts_free_read_env(void *value);

#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
extern void sys_thr_resume(erts_tid_t tid);
extern void sys_thr_suspend(erts_tid_t tid);
#endif
# 878 "beam/sys.h"

/* utils.c */

/* Options to sys_alloc_opt */
#define SYS_ALLOC_OPT_TRIM_THRESHOLD 0
#define SYS_ALLOC_OPT_TOP_PAD        1
#define SYS_ALLOC_OPT_MMAP_THRESHOLD 2
#define SYS_ALLOC_OPT_MMAP_MAX       3

/* Default values to sys_alloc_opt options */
#define ERTS_DEFAULT_TRIM_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_TOP_PAD         0
#define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_MMAP_MAX        64

int sys_alloc_opt(int, int);

typedef struct {
  int trim_threshold;
  int top_pad;
  int mmap_threshold;
  int mmap_max;
} SysAllocStat;

void sys_alloc_stat(SysAllocStat *);

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#undef ERTS_REFC_DEBUG
#define ERTS_REFC_DEBUG
#endif
# 908 "beam/sys.h"

typedef erts_smp_atomic_t erts_refc_t;

ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
				   erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
					   erts_aint_t min_val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}

ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 941 "beam/sys.h"
    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
# 943 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 955 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 968 "beam/sys.h"
    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
# 970 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 982 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
		 diff, val, min_val);
#else
# 995 "beam/sys.h"
    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
# 997 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 1009 "beam/sys.h"
    return val;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1013 "beam/sys.h"

#ifdef ERTS_ENABLE_KERNEL_POLL
extern int erts_use_kernel_poll;
#endif
# 1017 "beam/sys.h"

#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
#define sys_memset(s,c,n)    memset(s,c,n)
#define sys_memzero(s, n)    memset(s,'\0',n)
#define sys_strcmp(s1,s2)    strcmp(s1,s2)
#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
#define sys_strcpy(s1,s2)    strcpy(s1,s2)
#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
#define sys_strlen(s)        strlen(s)

/* define function symbols (needed in sys_drv_api) */
#define sys_fp_alloc     sys_alloc
#define sys_fp_realloc   sys_realloc
#define sys_fp_free      sys_free
#define sys_fp_memcpy    memcpy
#define sys_fp_memmove   memmove
#define sys_fp_memcmp    memcmp
#define sys_fp_memset    memset
/* #define sys_fp_memzero    elib_memzero */
#define sys_fp_strcmp    strcmp
#define sys_fp_strncmp   strncmp
#define sys_fp_strcpy    strcpy
#define sys_fp_strncpy   strncpy
#define sys_fp_strlen    strlen


/* Return codes from the nb_read and nb_write functions */
#define FD_READY 1
#define FD_CONTINUE 2
#define FD_ERROR 3



/* Standard set of integer macros  .. */

#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
                      (((Uint64)((unsigned char*) (s))[7])))

#define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                            ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
                            ((char*)(s))[2] = (char)((Sint64)(i) >> 40) & 0xff;\
                            ((char*)(s))[3] = (char)((Sint64)(i) >> 32) & 0xff;\
                            ((char*)(s))[4] = (char)((Sint64)(i) >> 24) & 0xff;\
                            ((char*)(s))[5] = (char)((Sint64)(i) >> 16) & 0xff;\
                            ((char*)(s))[6] = (char)((Sint64)(i) >> 8)  & 0xff;\
                            ((char*)(s))[7] = (char)((Sint64)(i))       & 0xff;\
                           } while (0) 

#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \
                      (((unsigned char*) (s))[1] << 16) | \
                      (((unsigned char*) (s))[2] << 8)  | \
                      (((unsigned char*) (s))[3]))

#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff;   \
                            ((char*)(s))[1] = (char)((i) >> 16) & 0xff;   \
                            ((char*)(s))[2] = (char)((i) >> 8)  & 0xff;   \
                            ((char*)(s))[3] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
                      (((unsigned char*) (s))[1] << 8)  | \
                      (((unsigned char*) (s))[2]))

#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                      (((unsigned char*)  (s))[1]))


#define put_int16(i, s) do {((char*)(s))[0] = (char)((i) >> 8) & 0xff;  \
                            ((char*)(s))[1] = (char)(i)        & 0xff;} \
                        while (0)

#define get_int8(s) ((((unsigned char*)  (s))[0] ))


#define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)


/*
 * Use DEBUGF as you would use printf, but use double parentheses:
 *
 *   DEBUGF(("Error: %s\n", error));
 *
 * The output will appear in a special console.
 */

#ifdef DEBUG
void erl_debug(char* format, ...);
void erl_bin_write(unsigned char *, int, int);

#  define DEBUGF(x) erl_debug x
#else
# 1121 "beam/sys.h"
#  define DEBUGF(x)
#endif
# 1123 "beam/sys.h"

#ifdef __WIN32__
#ifdef ARCH_64
#define ERTS_ALLOC_ALIGN_BYTES 16
#define ERTS_SMALL_ABS(Small) _abs64(Small) 
#else
# 1129 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1132 "beam/sys.h"
#else
# 1133 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1136 "beam/sys.h"

#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
#endif
# 1140 "beam/sys.h"

#ifdef __WIN32__
void call_break_handler(void);
char* last_error(void);
char* win32_errorstr(int);
#endif
# 1146 "beam/sys.h"

/************************************************************************
 * Find out the native filename encoding of the process (look at locale of 
 * Unix processes and just do UTF16 on windows 
 ************************************************************************/
#define ERL_FILENAME_UNKNOWN   (0)
#define ERL_FILENAME_LATIN1    (1)
#define ERL_FILENAME_UTF8      (2)
#define ERL_FILENAME_UTF8_MAC  (3)
#define ERL_FILENAME_WIN_WCHAR (4)

/************************************************************************
 * If a filename in for example list_dir is not in the right encoding, it
 * will be skipped in the resulting list, but depending on a startup setting
 * we will inform the user in different ways. These macros define the
 * different reactions to wrongly coded filenames. In the error case an
 * exception will be thrown by prim_file.
 ************************************************************************/
#define ERL_FILENAME_WARNING_WARNING (0)
#define ERL_FILENAME_WARNING_IGNORE (1)
#define ERL_FILENAME_WARNING_ERROR (2)

/***********************************************************************
 * The user can request a range of character that he/she consider
 * printable. Currently this can be either latin1 or unicode, but
 * in the future a set of ranges, or languages, could be specified.
 ***********************************************************************/
#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0)
#define ERL_PRINTABLE_CHARACTERS_UNICODE (1)

int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
void erts_set_user_requested_filename_encoding(int encoding, int warning);
int erts_get_user_requested_filename_encoding(void);
int erts_get_filename_warning_type(void);
/* This function is called from erl_init. The setting is read by BIF's 
   in io/io_lib. Setting is not atomic. */
void erts_set_printable_characters(int range);
/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */
int erts_get_printable_characters(void);

void erts_init_sys_common_misc(void);

#endif
# 1190 "beam/sys.h"
# 64 "beam/erl_lock_count.h" 2

#ifndef ERTS_LOCK_COUNT_H__
#define ERTS_LOCK_COUNT_H__

#ifdef  ERTS_ENABLE_LOCK_COUNT
#ifndef ERTS_ENABLE_LOCK_POSITION
/* Enable in order for _x variants of mtx functions to be used. */
#define ERTS_ENABLE_LOCK_POSITION 1
#endif
# 73 "beam/erl_lock_count.h"

#if 0 /* expanded by -frewrite-includes */
#include "ethread.h"
#endif /* expanded by -frewrite-includes */
# 74 "beam/erl_lock_count.h"
# 75 "beam/erl_lock_count.h"

#define ERTS_LCNT_MAX_LOCK_LOCATIONS  (10)

/* histogram */
#define ERTS_LCNT_HISTOGRAM_MAX_NS    (((unsigned long)1LL << 28) - 1)
#if 0 || defined(ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT)
#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (30)
#define ERTS_LCNT_HISTOGRAM_RSHIFT    (0)
#else
# 84 "beam/erl_lock_count.h"
#define ERTS_LCNT_HISTOGRAM_SLOT_SIZE (20)
#define ERTS_LCNT_HISTOGRAM_RSHIFT    (10)
#endif
# 87 "beam/erl_lock_count.h"

#define ERTS_LCNT_LT_SPINLOCK   (((Uint16) 1) << 0)
#define ERTS_LCNT_LT_RWSPINLOCK (((Uint16) 1) << 1)
#define ERTS_LCNT_LT_MUTEX      (((Uint16) 1) << 2)
#define ERTS_LCNT_LT_RWMUTEX    (((Uint16) 1) << 3)
#define ERTS_LCNT_LT_PROCLOCK   (((Uint16) 1) << 4)
#define ERTS_LCNT_LT_ALLOC      (((Uint16) 1) << 5)

#define ERTS_LCNT_LO_READ       (((Uint16) 1) << 6)
#define ERTS_LCNT_LO_WRITE      (((Uint16) 1) << 7)

#define ERTS_LCNT_LO_READ_WRITE ( ERTS_LCNT_LO_READ  \
                                | ERTS_LCNT_LO_WRITE )

#define ERTS_LCNT_LT_ALL        ( ERTS_LCNT_LT_SPINLOCK   \
                                | ERTS_LCNT_LT_RWSPINLOCK \
                                | ERTS_LCNT_LT_MUTEX      \
                                | ERTS_LCNT_LT_RWMUTEX    \
                                | ERTS_LCNT_LT_PROCLOCK   )

#define ERTS_LCNT_LOCK_TYPE(lock)       ((lock)->flag & ERTS_LCNT_LT_ALL)
#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL))
#define ERTS_LCNT_CLEAR_FLAG(lock)      ((lock)->flag = 0)

/* runtime options */

#define ERTS_LCNT_OPT_SUSPEND  (((Uint16) 1) << 0)
#define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1)
#define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2)
#define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3)
#define ERTS_LCNT_OPT_COPYSAVE (((Uint16) 1) << 4)

typedef struct {
    unsigned long s;
    unsigned long ns;
} erts_lcnt_time_t;

extern erts_lcnt_time_t timer_start;

typedef struct {
    Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */
} erts_lcnt_hist_t;

typedef struct erts_lcnt_lock_stats_s {
    /* "tries" and "colls" needs to be atomic since
     * trylock busy does not aquire a lock and there
     * is no post action to rectify the situation
     */

    char         *file;       /* which file the lock was taken */
    unsigned int  line;       /* line number in file */

    ethr_atomic_t tries;      /* n tries to get lock */
    ethr_atomic_t colls;      /* n collisions of tries to get lock */

    unsigned long timer_n;    /* #times waited for lock */
    erts_lcnt_time_t timer;   /* total wait time for lock */
    erts_lcnt_hist_t hist;
} erts_lcnt_lock_stats_t;

/* rw locks uses both states, other locks only uses w_state */
typedef struct erts_lcnt_lock_s {
    char *name;            /* lock name */
    Uint16 flag;           /* lock type */
    Eterm id;              /* id if possible */ 

#ifdef DEBUG
    ethr_atomic_t flowstate;
#endif
# 156 "beam/erl_lock_count.h"

    /* lock states */    
    ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */
    ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */

    /* statistics */
    unsigned int n_stats;
    erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/

    /* chains for list handling  */
    /* data is hold by lcnt_lock */
    struct erts_lcnt_lock_s *prev;
    struct erts_lcnt_lock_s *next;
} erts_lcnt_lock_t;

typedef struct {
    erts_lcnt_lock_t *head;
    erts_lcnt_lock_t *tail;
    unsigned long n;
} erts_lcnt_lock_list_t;

typedef struct {
    erts_lcnt_time_t duration;			/* time since last clear */
    erts_lcnt_lock_list_t *current_locks;
    erts_lcnt_lock_list_t *deleted_locks;
} erts_lcnt_data_t;

typedef struct {
    int id;

    erts_lcnt_time_t timer;	/* timer         */
    int timer_set;		/* bool          */
    int lock_in_conflict;       /* bool          */
} erts_lcnt_thread_data_t;

/* globals */

extern Uint16 erts_lcnt_rt_options;

/* function declerations */

void erts_lcnt_init(void);
void erts_lcnt_late_init(void);

/* thread operations */
void erts_lcnt_thread_setup(void);
void erts_lcnt_thread_exit_handler(void);

/* list operations (local)  */
erts_lcnt_lock_list_t *erts_lcnt_list_init(void);

void erts_lcnt_list_clear( erts_lcnt_lock_list_t *list);
void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);
void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock);

/* lock operations (global) */
void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag);
void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id);
void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock);
void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock);

void erts_lcnt_lock(erts_lcnt_lock_t *lock);
void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option);
void erts_lcnt_lock_post(erts_lcnt_lock_t *lock);
void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line);
void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock);

void erts_lcnt_unlock(erts_lcnt_lock_t *lock);
void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option);

void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option);
void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res);

/* bif interface */
Uint16 erts_lcnt_set_rt_opt(Uint16 opt);
Uint16 erts_lcnt_clear_rt_opt(Uint16 opt);
void   erts_lcnt_clear_counters(void);
char  *erts_lcnt_lock_type(Uint16 type);
erts_lcnt_data_t *erts_lcnt_get_data(void);

#endif /* ifdef  ERTS_ENABLE_LOCK_COUNT  */
# 237 "beam/erl_lock_count.h"
#endif /* ifndef ERTS_LOCK_COUNT_H__     */
# 238 "beam/erl_lock_count.h"
# 268 "beam/erl_threads.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 268 "beam/erl_threads.h"
# 1 "beam/erl_term.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2000-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERL_TERM_H
#define __ERL_TERM_H

#if 0 /* expanded by -frewrite-includes */
#include "sys.h" /* defines HALFWORD_HEAP */
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_term.h"
# 1 "beam/sys.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __SYS_H__
#define __SYS_H__

#ifdef ERTS_INLINE
#  ifndef ERTS_CAN_INLINE
#    define ERTS_CAN_INLINE 1
#  endif
# 28 "beam/sys.h"
#else
# 29 "beam/sys.h"
#  if defined(__GNUC__)
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline__
#  elif defined(__WIN32__)
# 33 "beam/sys.h"
#    define ERTS_CAN_INLINE 1
#    define ERTS_INLINE __inline
#  else
# 36 "beam/sys.h"
#    define ERTS_CAN_INLINE 0
#    define ERTS_INLINE
#  endif
# 39 "beam/sys.h"
#endif
# 40 "beam/sys.h"

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#  undef ERTS_CAN_INLINE
#  define ERTS_CAN_INLINE 0
#  undef ERTS_INLINE
#  define ERTS_INLINE
#endif
# 47 "beam/sys.h"

#if ERTS_CAN_INLINE
#define ERTS_GLB_INLINE static ERTS_INLINE
#else
# 51 "beam/sys.h"
#define ERTS_GLB_INLINE
#endif
# 53 "beam/sys.h"

#if ERTS_CAN_INLINE || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF) 
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 1
#else
# 57 "beam/sys.h"
#  define ERTS_GLB_INLINE_INCL_FUNC_DEF 0
#endif
# 59 "beam/sys.h"

#if defined(VALGRIND) && !defined(NO_FPE_SIGNALS)
#  define NO_FPE_SIGNALS
#endif
# 63 "beam/sys.h"

#ifdef DISABLE_CHILD_WAITER_THREAD
#undef ENABLE_CHILD_WAITER_THREAD
#endif
# 67 "beam/sys.h"

#if defined(ERTS_SMP) && !defined(DISABLE_CHILD_WAITER_THREAD)
#undef ENABLE_CHILD_WAITER_THREAD
#define ENABLE_CHILD_WAITER_THREAD 1
#endif
# 72 "beam/sys.h"

#define ERTS_I64_LITERAL(X) X##LL

#if defined (__WIN32__)
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_sys.h"
#endif /* expanded by -frewrite-includes */
# 76 "beam/sys.h"
# 77 "beam/sys.h"
#elif defined (__OSE__)
# 78 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_ose_sys.h"
#endif /* expanded by -frewrite-includes */
# 78 "beam/sys.h"
# 79 "beam/sys.h"
#else 
# 80 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#  include "erl_unix_sys.h"
#endif /* expanded by -frewrite-includes */
# 80 "beam/sys.h"
# 81 "beam/sys.h"
#ifndef UNIX
#  define UNIX 1
#endif
# 84 "beam/sys.h"
#endif
# 85 "beam/sys.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_misc_utils.h"
#endif /* expanded by -frewrite-includes */
# 86 "beam/sys.h"
# 87 "beam/sys.h"

/*
 * To allow building of Universal Binaries for Mac OS X,
 * we must not depend on the endian detected by the configure script.
 */
#if defined(__APPLE__)
#  if defined(__BIG_ENDIAN__) && !defined(WORDS_BIGENDIAN)
#    define WORDS_BIGENDIAN 1
#  elif !defined(__BIG_ENDIAN__) && defined(WORDS_BIGENDIAN)
# 96 "beam/sys.h"
#    undef WORDS_BIGENDIAN
#  endif
# 98 "beam/sys.h"
#endif
# 99 "beam/sys.h"

/*
 * Make sure we have a type for FD's (used by erl_check_io)
 */

#ifndef ERTS_SYS_FD_TYPE
#define ERTS_SYS_FD_INVALID ((ErtsSysFdType) -1)
typedef int ErtsSysFdType;
#else
# 108 "beam/sys.h"
#ifndef ERTS_SYS_FD_INVALID
# error missing ERTS_SYS_FD_INVALID
#endif
# 111 "beam/sys.h"
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
# 113 "beam/sys.h"

#if !defined(__GNUC__)
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
#elif !defined(__GNUC_MINOR__)
# 117 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#elif !defined(__GNUC_PATCHLEVEL__)
# 120 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#else
# 123 "beam/sys.h"
#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
#endif
# 126 "beam/sys.h"

#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
#  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
#  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)
#else
# 131 "beam/sys.h"
#  define ERTS_LIKELY(BOOL)   (BOOL)
#  define ERTS_UNLIKELY(BOOL) (BOOL)
#endif
# 134 "beam/sys.h"
#ifdef __GNUC__
#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 5)
#    define ERTS_DECLARE_DUMMY(X) X __attribute__ ((unused))
#  else
# 138 "beam/sys.h"
#    define ERTS_DECLARE_DUMMY(X) X
#  endif
# 140 "beam/sys.h"
#else
# 141 "beam/sys.h"
#  define ERTS_DECLARE_DUMMY(X) X
#endif
# 143 "beam/sys.h"

#if !defined(__func__)
#  if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L
#    if !defined(__GNUC__) ||  __GNUC__ < 2
#      define __func__ "[unknown_function]"
#    else
# 149 "beam/sys.h"
#      define __func__ __FUNCTION__
#    endif
# 151 "beam/sys.h"
#  endif
# 152 "beam/sys.h"
#endif
# 153 "beam/sys.h"

#define ERTS_MK_VSN_INT(Major, Minor, Build) \
    ((((Major) & 0x3ff) << 20) | (((Minor) & 0x3ff) << 10) | ((Build) & 0x3ff))

#ifndef ERTS_EXIT_AFTER_DUMP
#  define ERTS_EXIT_AFTER_DUMP exit
#endif
# 160 "beam/sys.h"

/* In VC++, noreturn is a declspec that has to be before the types,
 * but in GNUC it is an att ribute to be placed between return type
 * and function name, hence __decl_noreturn <types> __noreturn <function name>
 *
 * at some platforms (e.g. Android) __noreturn is defined at sys/cdef.h
 */
#if __GNUC__
#  define __decl_noreturn
#  ifndef __noreturn
#     define __noreturn __attribute__((noreturn))
#  endif
# 172 "beam/sys.h"
#else
# 173 "beam/sys.h"
#  if defined(__WIN32__) && defined(_MSC_VER)
#    define __noreturn
#    define __decl_noreturn __declspec(noreturn)
#  else
# 177 "beam/sys.h"
#    define __noreturn
#    define __decl_noreturn
#  endif
# 180 "beam/sys.h"
#endif
# 181 "beam/sys.h"

#define ERTS_ASSERT(e) \
    ((void) ((e) ? 1 : (erl_assert_error(#e, __func__, __FILE__, __LINE__), 0)))

__decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *func,
						 const char* file, int line);

#ifdef DEBUG
#  define ASSERT(e) ERTS_ASSERT(e)
#else
# 191 "beam/sys.h"
#  define ASSERT(e) ((void) 1)
#endif
# 193 "beam/sys.h"

/* ERTS_UNDEF can be used to silence false warnings about
 * "variable may be used uninitialized" while keeping the variable
 * marked as undefined by valgrind.
 */
#ifdef VALGRIND
#  define ERTS_UNDEF(V,I)
#else
# 201 "beam/sys.h"
#  define ERTS_UNDEF(V,I) V = I
#endif
# 203 "beam/sys.h"

/*
 * Compile time assert
 * (the actual compiler error msg can be a bit confusing)
 */
#if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
# define ERTS_CT_ASSERT(e) \
    do { \
	enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \
    } while(0)
#else
# 214 "beam/sys.h"
# define ERTS_CT_ASSERT(e) \
    do { \
        enum { compile_time_assert__ = 1/(e) }; \
    } while (0)
#endif
# 219 "beam/sys.h"

/*
 * Microsoft C/C++: We certainly want to use stdarg.h and prototypes.
 * But MSC doesn't define __STDC__, unless we compile with the -Za
 * flag (strict ANSI C, no Microsoft extension).  Compiling with -Za
 * doesn't work: some Microsoft headers fail to compile...
 *
 * Solution: Test if __STDC__ or _MSC_VER is defined.
 *
 * Note: Simply defining __STDC__ doesn't work, as some Microsoft
 * headers will fail to compile!
 */

#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 232 "beam/sys.h"
# 233 "beam/sys.h"

/* This isn't sys-dependent, but putting it here benefits sys.c and drivers
   - allow use of 'const' regardless of compiler */

#if !defined(__STDC__) && !defined(_MSC_VER)
#  define const
#endif
# 240 "beam/sys.h"

#undef __deprecated
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 0)
#  define __deprecated __attribute__((deprecated))
#else
# 245 "beam/sys.h"
#  define __deprecated
#endif
# 247 "beam/sys.h"
#if ERTS_AT_LEAST_GCC_VSN__(3, 0, 4)
#  define erts_align_attribute(SZ) __attribute__ ((aligned (SZ)))
#else
# 250 "beam/sys.h"
#  define erts_align_attribute(SZ)
#endif
# 252 "beam/sys.h"

/*
** Data types:
**
** Eterm: A tagged erlang term (possibly 64 bits)
** BeamInstr: A beam code instruction unit, possibly larger than Eterm, not smaller.
** UInt:  An unsigned integer exactly as large as an Eterm.
** SInt:  A signed integer exactly as large as an eterm and therefor large
**        enough to hold the return value of the signed_val() macro.
** UWord: An unsigned integer at least as large as a void * and also as large
**          or larger than an Eterm
** SWord: A signed integer at least as large as a void * and also as large
**          or larger than an Eterm
** Uint32: An unsigned integer of 32 bits exactly
** Sint32: A signed integer of 32 bits exactly
** Uint16: An unsigned integer of 16 bits exactly
** Sint16: A signed integer of 16 bits exactly.
*/

#if !((SIZEOF_VOID_P >= 4) && (SIZEOF_VOID_P == SIZEOF_SIZE_T) \
      && ((SIZEOF_VOID_P == SIZEOF_INT) || (SIZEOF_VOID_P == SIZEOF_LONG) || \
          (SIZEOF_VOID_P == SIZEOF_LONG_LONG)))
#error Cannot handle this combination of int/long/void*/size_t sizes
#endif
# 276 "beam/sys.h"

#if SIZEOF_VOID_P == 8
#undef  ARCH_32
#define ARCH_64
#define ERTS_SIZEOF_TERM 8
#elif SIZEOF_VOID_P == 4
# 282 "beam/sys.h"
#define ARCH_32
#undef  ARCH_64
#define ERTS_SIZEOF_TERM 4
#else
# 286 "beam/sys.h"
#error Neither 32 nor 64 bit architecture
#endif
# 288 "beam/sys.h"
#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR)
#    define HALFWORD_HEAP 1
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND) ASSERT(COND)
#    undef ERTS_SIZEOF_TERM
#    define ERTS_SIZEOF_TERM 4
#else
# 295 "beam/sys.h"
#    define HALFWORD_HEAP 0
#    define HALFWORD_ASSERT 0
#    define ASSERT_HALFWORD(COND)
#endif
# 299 "beam/sys.h"

#if SIZEOF_VOID_P != SIZEOF_SIZE_T
#error sizeof(void*) != sizeof(size_t)
#endif
# 303 "beam/sys.h"

#if HALFWORD_HEAP

#if SIZEOF_INT == 4
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define ERTS_UINT_MAX UINT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#else
# 314 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 316 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long UWord;
typedef long          SWord;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#elif SIZEOF_VOID_P == SIZEOF_INT
# 325 "beam/sys.h"
typedef unsigned int UWord;
typedef int          SWord;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 332 "beam/sys.h"
typedef unsigned long long UWord;
typedef long long          SWord;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#else
# 339 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 341 "beam/sys.h"

#else /* !HALFWORD_HEAP */
# 343 "beam/sys.h"

#if SIZEOF_VOID_P == SIZEOF_LONG
typedef unsigned long Eterm;
typedef unsigned long Uint;
typedef long          Sint;
#define SWORD_CONSTANT(Const) Const##L
#define UWORD_CONSTANT(Const) Const##UL
#define ERTS_UWORD_MAX ULONG_MAX
#define ERTS_SWORD_MAX LONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_INT
# 355 "beam/sys.h"
typedef unsigned int Eterm;
typedef unsigned int Uint;
typedef int          Sint;
#define SWORD_CONSTANT(Const) Const
#define UWORD_CONSTANT(Const) Const##U
#define ERTS_UWORD_MAX UINT_MAX
#define ERTS_SWORD_MAX INT_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_INT
#define ErtsStrToSint strtol
#elif SIZEOF_VOID_P == SIZEOF_LONG_LONG
# 365 "beam/sys.h"
typedef unsigned long long Eterm;
typedef unsigned long long Uint;
typedef long long          Sint;
#define SWORD_CONSTANT(Const) Const##LL
#define UWORD_CONSTANT(Const) Const##ULL
#define ERTS_UWORD_MAX ULLONG_MAX
#define ERTS_SWORD_MAX LLONG_MAX
#define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG
#if defined(__WIN32__)
#define ErtsStrToSint _strtoi64
#else
# 376 "beam/sys.h"
#define ErtsStrToSint strtoll
#endif
# 378 "beam/sys.h"
#else
# 379 "beam/sys.h"
#error Found no appropriate type to use for 'Eterm', 'Uint' and 'Sint'
#endif
# 381 "beam/sys.h"

typedef Uint UWord;
typedef Sint SWord;
#define ERTS_UINT_MAX ERTS_UWORD_MAX

#endif /* HALFWORD_HEAP */
# 387 "beam/sys.h"

typedef UWord BeamInstr;

#ifndef HAVE_INT64
#  if SIZEOF_LONG == 8
#    define HAVE_INT64 1
typedef unsigned long Uint64;
typedef long          Sint64;
#    ifdef ULONG_MAX
#      define ERTS_UINT64_MAX ULONG_MAX
#    endif
# 398 "beam/sys.h"
#    ifdef LONG_MAX
#      define ERTS_SINT64_MAX LONG_MAX
#    endif
# 401 "beam/sys.h"
#    ifdef LONG_MIN
#      define ERTS_SINT64_MIN LONG_MIN
#    endif
# 404 "beam/sys.h"
#  elif SIZEOF_LONG_LONG == 8
# 405 "beam/sys.h"
#    define HAVE_INT64 1
typedef unsigned long long Uint64;
typedef long long          Sint64;
#    ifdef ULLONG_MAX
#      define ERTS_UINT64_MAX ULLONG_MAX
#    endif
# 411 "beam/sys.h"
#    ifdef LLONG_MAX
#      define ERTS_SINT64_MAX LLONG_MAX
#    endif
# 414 "beam/sys.h"
#    ifdef LLONG_MIN
#      define ERTS_SINT64_MIN LLONG_MIN
#    endif
# 417 "beam/sys.h"
#  else
# 418 "beam/sys.h"
#    error "No 64-bit integer type found"
#  endif
# 420 "beam/sys.h"
#endif
# 421 "beam/sys.h"

#ifndef ERTS_UINT64_MAX
#  define ERTS_UINT64_MAX (~((Uint64) 0))
#endif
# 425 "beam/sys.h"
#ifndef ERTS_SINT64_MAX
#  define ERTS_SINT64_MAX ((Sint64) ((((Uint64) 1) << 63)-1))
#endif
# 428 "beam/sys.h"
#ifndef ERTS_SINT64_MIN
#  define ERTS_SINT64_MIN (-1*(((Sint64) 1) << 63))
#endif
# 431 "beam/sys.h"

#if SIZEOF_LONG == 4
typedef unsigned long Uint32;
typedef long          Sint32;
#elif SIZEOF_INT == 4
# 436 "beam/sys.h"
typedef unsigned int Uint32;
typedef int          Sint32;
#else
# 439 "beam/sys.h"
#error Found no appropriate type to use for 'Uint32' and 'Sint32'
#endif
# 441 "beam/sys.h"

#if SIZEOF_INT == 2
typedef unsigned int Uint16;
typedef int          Sint16;
#elif SIZEOF_SHORT == 2
# 446 "beam/sys.h"
typedef unsigned short Uint16;
typedef short          Sint16;
#else
# 449 "beam/sys.h"
#error Found no appropriate type to use for 'Uint16' and 'Sint16'
#endif
# 451 "beam/sys.h"

#if CHAR_BIT == 8
typedef unsigned char byte;
#else
# 455 "beam/sys.h"
#error Found no appropriate type to use for 'byte'
#endif
# 457 "beam/sys.h"

#if defined(ARCH_64) && !HAVE_INT64
#error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found 
#endif
# 461 "beam/sys.h"

#ifdef WORDS_BIGENDIAN
#  define ERTS_HUINT_HVAL_HIGH 0
#  define ERTS_HUINT_HVAL_LOW 1
#else
# 466 "beam/sys.h"
#  define ERTS_HUINT_HVAL_HIGH 1
#  define ERTS_HUINT_HVAL_LOW 0
#endif
# 469 "beam/sys.h"
#if ERTS_SIZEOF_TERM == 8
typedef union {
    Uint val;
    Uint32 hval[2];
} HUint;
#elif ERTS_SIZEOF_TERM == 4
# 475 "beam/sys.h"
typedef union {
    Uint val;
    Uint16 hval[2];
} HUint;
#else
# 480 "beam/sys.h"
#error "Unsupported size of term"
#endif
# 482 "beam/sys.h"

#  define ERTS_EXTRA_DATA_ALIGN_SZ(X) \
    (((size_t) 8) - (((size_t) (X)) & ((size_t) 7)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 486 "beam/sys.h"
# 487 "beam/sys.h"

/* needed by erl_smp.h */
int erts_send_warning_to_logger_str_nogl(char *);

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/sys.h"
# 492 "beam/sys.h"

#ifdef ERTS_WANT_BREAK_HANDLING
#  ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#    define ERTS_BREAK_REQUESTED \
  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
#  else
# 499 "beam/sys.h"
extern volatile int erts_break_requested;
#    define ERTS_BREAK_REQUESTED erts_break_requested
#  endif
# 502 "beam/sys.h"
void erts_do_break_handling(void);
#endif
# 504 "beam/sys.h"

#ifdef ERTS_WANT_GOT_SIGUSR1
#  ifndef UNIX
#    define ERTS_GOT_SIGUSR1 0
#  else
# 509 "beam/sys.h"
#    ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
#    else
# 513 "beam/sys.h"
extern volatile int erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
#    endif
# 516 "beam/sys.h"
#  endif
# 517 "beam/sys.h"
#endif
# 518 "beam/sys.h"

#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
extern erts_tsd_key_t erts_is_crash_dumping_key;
#define ERTS_SOMEONE_IS_CRASH_DUMPING \
  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
# 527 "beam/sys.h"
extern volatile int erts_writing_erl_crash_dump;
#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
# 531 "beam/sys.h"

/* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
   but be able to fall back on bcopy() etc on systems that don't have
   mem*(), but this doesn't work to well with memset()/bzero() - thus the
   memzero() macro.
*/

/* xxxP */
#if defined(USE_BCOPY)
#  define memcpy(a, b, c) bcopy((b), (a), (c))
#  define memcmp(a, b, c) bcmp((a), (b), (c))
#  define memzero(buf, len) bzero((buf), (len))
#else
# 544 "beam/sys.h"
#  define memzero(buf, len) memset((buf), '\0', (len))
#endif
# 546 "beam/sys.h"

/* Stuff that is useful for port programs, drivers, etc */

#ifdef ISC32			/* Too much for the Makefile... */
#  define signal	sigset
#  define NO_ASINH
#  define NO_ACOSH
#  define NO_ATANH
#  define NO_FTRUNCATE
#  define SIG_SIGHOLD
#  define _POSIX_SOURCE 
#  define _XOPEN_SOURCE
#endif
# 559 "beam/sys.h"

#ifdef QNX			/* Too much for the Makefile... */
#  define SYS_SELECT_H
#  define NO_ERF
#  define NO_ERFC
/* This definition doesn't take NaN into account, but matherr() gets those */
#  define isfinite(x) (fabs(x) != HUGE_VAL)
#  define USE_MATHERR
#  define HAVE_FINITE
#endif
# 569 "beam/sys.h"


#ifdef WANT_NONBLOCKING	    /* must define this to pull in fcntl.h/ioctl.h */

/* This is really a mess... We used to use fcntl O_NDELAY, but that seems
   to only work on SunOS 4 - in particular, on SysV-based systems
   (including Solaris 2), it does set non-blocking mode, but causes
   read() to return 0!!  fcntl O_NONBLOCK is specified by POSIX, and
   seems to work on most systems, with the notable exception of AIX,
   where the old ioctl FIONBIO is the *only* one that will set a *socket*
   in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
   pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */

#  ifdef __WIN32__

static unsigned long zero_value = 0, one_value = 1;
#    define SET_BLOCKING(fd)	{ if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); }
#    define SET_NONBLOCKING(fd)	ioctlsocket((fd), FIONBIO, &one_value)

#  else
# 589 "beam/sys.h"
#    ifdef NB_FIONBIO		/* Old BSD */
#if 0 /* expanded by -frewrite-includes */
#      include <sys/ioctl.h>
#endif /* expanded by -frewrite-includes */
# 590 "beam/sys.h"
# 591 "beam/sys.h"
  static const int zero_value = 0, one_value = 1;
#      define SET_BLOCKING(fd)         ioctl((fd), FIONBIO, &zero_value)
#      define SET_NONBLOCKING(fd)      ioctl((fd), FIONBIO, &one_value)
#      define ERRNO_BLOCK EWOULDBLOCK
#    else /* !NB_FIONBIO */
# 596 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#      include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 596 "beam/sys.h"
# 597 "beam/sys.h"
#      ifdef NB_O_NDELAY               /* Nothing needs this? */
#        define NB_FLAG O_NDELAY
#        ifndef ERRNO_BLOCK            /* allow override (e.g. EAGAIN) via Makefile */
#          define ERRNO_BLOCK EWOULDBLOCK
#        endif
# 602 "beam/sys.h"
#      else  /* !NB_O_NDELAY */	/* The True Way - POSIX!:-) */
# 603 "beam/sys.h"
#        define NB_FLAG O_NONBLOCK
#        define ERRNO_BLOCK EAGAIN
#      endif /* !NB_O_NDELAY */
# 606 "beam/sys.h"
#      define SET_BLOCKING(fd)         fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) & ~NB_FLAG)
#      define SET_NONBLOCKING(fd)      fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) | NB_FLAG)
#    endif /* !NB_FIONBIO */
# 611 "beam/sys.h"
#  endif /* !__WIN32__ */
# 612 "beam/sys.h"
#endif /* WANT_NONBLOCKING */
# 613 "beam/sys.h"

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);

/* Some special erts_exit() codes: */
#define ERTS_INTR_EXIT	-1		/* called from signal handler */
#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */

#define ERTS_INTERNAL_ERROR(What) \
    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
	     __FILE__, __LINE__, __func__, What)

Eterm erts_check_io_info(void *p);

/* Size of misc memory allocated from system dependent code */
Uint erts_sys_misc_mem_sz(void);

/* print stuff is declared here instead of in global.h, so sys stuff won't
   have to include global.h */
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 633 "beam/sys.h"
# 634 "beam/sys.h"

/* Io constants to erts_print and erts_putc */
#define ERTS_PRINT_STDERR	(2)
#define ERTS_PRINT_STDOUT	(1)
#define ERTS_PRINT_FILE		(-1)
#define ERTS_PRINT_SBUF		(-2)
#define ERTS_PRINT_SNBUF	(-3)
#define ERTS_PRINT_DSBUF	(-4)

#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF

typedef struct {
    char *buf;
    size_t size;
} erts_print_sn_buf;

int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
int erts_putc(int to, void *arg, char);			/* in utils.c */

/* logger stuff is declared here instead of in global.h, so sys files
   won't have to include global.h */

erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
int erts_send_info_to_logger_str(Eterm, char *); 
int erts_send_warning_to_logger_str(Eterm, char *);
int erts_send_error_to_logger_str(Eterm, char *);
int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
/* needed by erl_smp.h (declared above)
   int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);

typedef struct preload {
    char *name;			/* Name of module */
    int  size;			/* Size of code */
    unsigned char* code;	/* Code pointer */
} Preload;


/*
 * This structure contains options to all built in drivers.
 * None of the drivers use all of the fields.
 */

typedef struct _SysDriverOpts {
    Uint ifd;			/* Input file descriptor (fd driver). */
    Uint ofd;			/* Outputfile descriptor (fd driver). */
    int packet_bytes;		/* Number of bytes in packet header. */
    int read_write;		/* Read and write bits. */
    int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
    int redir_stderr;           /* Redirect stderr to stdout: TRUE/FALSE. */
    int hide_window;		/* Hide this windows (Windows). */
    int exit_status;		/* Report exit status of subprocess. */
    int overlapped_io;          /* Only has effect on windows NT et al */
    char *envir;		/* Environment of the port process, */
				/* in Windows format. */
    char **argv;                /* Argument vector in Unix'ish format. */
    char *wd;			/* Working directory. */
    unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
				   ERTS_SPAWN_EXTERNAL | both*/ 
    int parallelism;            /* Optimize for parallelism */
} SysDriverOpts;

extern char *erts_default_arg0;

extern char os_type[];

typedef struct {
    int have_os_monotonic_time;
    int have_corrected_os_monotonic_time;
    ErtsMonotonicTime os_monotonic_time_unit;
    ErtsMonotonicTime sys_clock_resolution;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
	int extended;
    } os_monotonic_time_info;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
    } os_system_time_info;
} ErtsSysInitTimeResult;

#define ERTS_SYS_INIT_TIME_RESULT_INITER \
    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}

extern void erts_init_sys_time_sup(void);
extern void sys_init_time(ErtsSysInitTimeResult *);
extern void erts_late_sys_init_time(void);
extern void erts_deliver_time(void);
extern void erts_time_remaining(SysTimeval *);
extern void erts_sys_init_float(void);
extern void erts_thread_init_float(void);
extern void erts_thread_disable_fpe(void);
ERTS_GLB_INLINE int erts_block_fpe(void);
ERTS_GLB_INLINE void erts_unblock_fpe(int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int erts_block_fpe(void)
{
    return erts_sys_block_fpe();
}

ERTS_GLB_INLINE void erts_unblock_fpe(int unmasked)
{
    erts_sys_unblock_fpe(unmasked);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 754 "beam/sys.h"


/* Dynamic library/driver loading */
typedef struct {
    char* str;
}ErtsSysDdllError;
#define ERTS_SYS_DDLL_ERROR_INIT {NULL}
extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*);
#define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL)
extern void *erts_sys_ddll_call_init(void *function);
extern void *erts_sys_ddll_call_nif_init(void *function);
extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*);
#define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL)
extern char *erts_sys_ddll_error(int code);



/*
 * System interfaces for startup.
 */
void erts_sys_schedule_interrupt(int set);
#ifdef ERTS_SMP
void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
void erts_sys_main_thread(void);
#endif
# 785 "beam/sys.h"

extern int erts_sys_prepare_crash_dump(int secs);
extern void erts_sys_pre_init(void);
extern void erl_sys_init(void);
extern void erl_sys_args(int *argc, char **argv);
extern void erl_sys_schedule(int);
void sys_tty_reset(int);

int sys_max_files(void);
void sys_init_io(void);
Preload* sys_preloaded(void);
unsigned char* sys_preload_begin(Preload*);
void sys_preload_end(Preload*);
int sys_get_key(int);
void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
		       UWord *ms_user_diff, UWord *ms_sys_diff);
void wall_clock_elapsed_time_both(UWord *ms_total, 
				  UWord *ms_diff);
void get_time(int *hour, int *minute, int *second);
void get_date(int *year, int *month, int *day);
void get_localtime(int *year, int *month, int *day, 
		   int *hour, int *minute, int *second);
void get_universaltime(int *year, int *month, int *day, 
		       int *hour, int *minute, int *second);
int seconds_to_univ(Sint64 seconds, 
		    Sint *year, Sint *month, Sint *day, 
		    Sint *hour, Sint *minute, Sint *second);
int univ_to_seconds(Sint year, Sint month, Sint day, 
		    Sint hour, Sint minute, Sint second,
		    Sint64* seconds);
int univ_to_local(
    Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second);
int local_to_univ(Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second, int isdst);
void get_now(Uint*, Uint*, Uint*);
struct ErtsSchedulerData_;
ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
ErtsMonotonicTime erts_get_time_offset(void);
void
erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
void get_sys_now(Uint*, Uint*, Uint*);
void set_break_quit(void (*)(void), void (*)(void));

void os_flavor(char*, unsigned);
void os_version(int*, int*, int*);
void init_getenv_state(GETENV_STATE *);
char * getenv_string(GETENV_STATE *);
void fini_getenv_state(GETENV_STATE *);

#define HAVE_ERTS_CHECK_IO_DEBUG
typedef struct {
    int no_used_fds;
    int no_driver_select_structs;
    int no_driver_event_structs;
} ErtsCheckIoDebugInfo;
int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);

int erts_sys_is_area_readable(char *start, char *stop);

/* xxxP */
#define SYS_DEFAULT_FLOAT_DECIMALS 20
void init_sys_float(void);
int sys_chars_to_double(char*, double*);
int sys_double_to_chars(double, char*, size_t);
int sys_double_to_chars_ext(double, char*, size_t, size_t);
int sys_double_to_chars_fast(double, char*, int, int, int);
void sys_get_pid(char *, size_t);

/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
int erts_sys_putenv(char *key, char *value);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_putenv_raw(char *key, char *value);
/* erts_sys_getenv() returns 0 on success (length of value string in
   *size), a value > 0 if value buffer is too small (*size is set to needed
   size), and a value < 0 on failure. */
int erts_sys_getenv(char *key, char *value, size_t *size);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_getenv_raw(char *key, char *value, size_t *size);
/* erts_sys_getenv__() is only allowed to be used in early init phase */
int erts_sys_getenv__(char *key, char *value, size_t *size);
/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
int erts_sys_unsetenv(char *key);

/* Easier to use, but not as efficient, environment functions */
char *erts_read_env(char *key);
void erts_free_read_env(void *value);

#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
extern void sys_thr_resume(erts_tid_t tid);
extern void sys_thr_suspend(erts_tid_t tid);
#endif
# 878 "beam/sys.h"

/* utils.c */

/* Options to sys_alloc_opt */
#define SYS_ALLOC_OPT_TRIM_THRESHOLD 0
#define SYS_ALLOC_OPT_TOP_PAD        1
#define SYS_ALLOC_OPT_MMAP_THRESHOLD 2
#define SYS_ALLOC_OPT_MMAP_MAX       3

/* Default values to sys_alloc_opt options */
#define ERTS_DEFAULT_TRIM_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_TOP_PAD         0
#define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_MMAP_MAX        64

int sys_alloc_opt(int, int);

typedef struct {
  int trim_threshold;
  int top_pad;
  int mmap_threshold;
  int mmap_max;
} SysAllocStat;

void sys_alloc_stat(SysAllocStat *);

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#undef ERTS_REFC_DEBUG
#define ERTS_REFC_DEBUG
#endif
# 908 "beam/sys.h"

typedef erts_smp_atomic_t erts_refc_t;

ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
				   erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
					   erts_aint_t min_val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}

ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 941 "beam/sys.h"
    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
# 943 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 955 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 968 "beam/sys.h"
    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
# 970 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 982 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
		 diff, val, min_val);
#else
# 995 "beam/sys.h"
    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
# 997 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 1009 "beam/sys.h"
    return val;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1013 "beam/sys.h"

#ifdef ERTS_ENABLE_KERNEL_POLL
extern int erts_use_kernel_poll;
#endif
# 1017 "beam/sys.h"

#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
#define sys_memset(s,c,n)    memset(s,c,n)
#define sys_memzero(s, n)    memset(s,'\0',n)
#define sys_strcmp(s1,s2)    strcmp(s1,s2)
#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
#define sys_strcpy(s1,s2)    strcpy(s1,s2)
#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
#define sys_strlen(s)        strlen(s)

/* define function symbols (needed in sys_drv_api) */
#define sys_fp_alloc     sys_alloc
#define sys_fp_realloc   sys_realloc
#define sys_fp_free      sys_free
#define sys_fp_memcpy    memcpy
#define sys_fp_memmove   memmove
#define sys_fp_memcmp    memcmp
#define sys_fp_memset    memset
/* #define sys_fp_memzero    elib_memzero */
#define sys_fp_strcmp    strcmp
#define sys_fp_strncmp   strncmp
#define sys_fp_strcpy    strcpy
#define sys_fp_strncpy   strncpy
#define sys_fp_strlen    strlen


/* Return codes from the nb_read and nb_write functions */
#define FD_READY 1
#define FD_CONTINUE 2
#define FD_ERROR 3



/* Standard set of integer macros  .. */

#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
                      (((Uint64)((unsigned char*) (s))[7])))

#define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                            ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
                            ((char*)(s))[2] = (char)((Sint64)(i) >> 40) & 0xff;\
                            ((char*)(s))[3] = (char)((Sint64)(i) >> 32) & 0xff;\
                            ((char*)(s))[4] = (char)((Sint64)(i) >> 24) & 0xff;\
                            ((char*)(s))[5] = (char)((Sint64)(i) >> 16) & 0xff;\
                            ((char*)(s))[6] = (char)((Sint64)(i) >> 8)  & 0xff;\
                            ((char*)(s))[7] = (char)((Sint64)(i))       & 0xff;\
                           } while (0) 

#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \
                      (((unsigned char*) (s))[1] << 16) | \
                      (((unsigned char*) (s))[2] << 8)  | \
                      (((unsigned char*) (s))[3]))

#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff;   \
                            ((char*)(s))[1] = (char)((i) >> 16) & 0xff;   \
                            ((char*)(s))[2] = (char)((i) >> 8)  & 0xff;   \
                            ((char*)(s))[3] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
                      (((unsigned char*) (s))[1] << 8)  | \
                      (((unsigned char*) (s))[2]))

#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                      (((unsigned char*)  (s))[1]))


#define put_int16(i, s) do {((char*)(s))[0] = (char)((i) >> 8) & 0xff;  \
                            ((char*)(s))[1] = (char)(i)        & 0xff;} \
                        while (0)

#define get_int8(s) ((((unsigned char*)  (s))[0] ))


#define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)


/*
 * Use DEBUGF as you would use printf, but use double parentheses:
 *
 *   DEBUGF(("Error: %s\n", error));
 *
 * The output will appear in a special console.
 */

#ifdef DEBUG
void erl_debug(char* format, ...);
void erl_bin_write(unsigned char *, int, int);

#  define DEBUGF(x) erl_debug x
#else
# 1121 "beam/sys.h"
#  define DEBUGF(x)
#endif
# 1123 "beam/sys.h"

#ifdef __WIN32__
#ifdef ARCH_64
#define ERTS_ALLOC_ALIGN_BYTES 16
#define ERTS_SMALL_ABS(Small) _abs64(Small) 
#else
# 1129 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1132 "beam/sys.h"
#else
# 1133 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1136 "beam/sys.h"

#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
#endif
# 1140 "beam/sys.h"

#ifdef __WIN32__
void call_break_handler(void);
char* last_error(void);
char* win32_errorstr(int);
#endif
# 1146 "beam/sys.h"

/************************************************************************
 * Find out the native filename encoding of the process (look at locale of 
 * Unix processes and just do UTF16 on windows 
 ************************************************************************/
#define ERL_FILENAME_UNKNOWN   (0)
#define ERL_FILENAME_LATIN1    (1)
#define ERL_FILENAME_UTF8      (2)
#define ERL_FILENAME_UTF8_MAC  (3)
#define ERL_FILENAME_WIN_WCHAR (4)

/************************************************************************
 * If a filename in for example list_dir is not in the right encoding, it
 * will be skipped in the resulting list, but depending on a startup setting
 * we will inform the user in different ways. These macros define the
 * different reactions to wrongly coded filenames. In the error case an
 * exception will be thrown by prim_file.
 ************************************************************************/
#define ERL_FILENAME_WARNING_WARNING (0)
#define ERL_FILENAME_WARNING_IGNORE (1)
#define ERL_FILENAME_WARNING_ERROR (2)

/***********************************************************************
 * The user can request a range of character that he/she consider
 * printable. Currently this can be either latin1 or unicode, but
 * in the future a set of ranges, or languages, could be specified.
 ***********************************************************************/
#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0)
#define ERL_PRINTABLE_CHARACTERS_UNICODE (1)

int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
void erts_set_user_requested_filename_encoding(int encoding, int warning);
int erts_get_user_requested_filename_encoding(void);
int erts_get_filename_warning_type(void);
/* This function is called from erl_init. The setting is read by BIF's 
   in io/io_lib. Setting is not atomic. */
void erts_set_printable_characters(int range);
/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */
int erts_get_printable_characters(void);

void erts_init_sys_common_misc(void);

#endif
# 1190 "beam/sys.h"
# 25 "beam/erl_term.h" 2

typedef UWord Wterm;  /* Full word terms */

#if HALFWORD_HEAP
#  define HEAP_ON_C_STACK 0
#  if HALFWORD_ASSERT
#    ifdef ET_DEBUG
#      undef ET_DEBUG
#    endif
# 34 "beam/erl_term.h"
#    define ET_DEBUG 1
#  endif
# 36 "beam/erl_term.h"
#  if 1
#    define CHECK_POINTER_MASK 0xFFFFFFFF00000000UL
#    define COMPRESS_POINTER(APointer) ((Eterm) (UWord) (APointer))
#    define EXPAND_POINTER(AnEterm) ((UWord) (AnEterm))
#  else
# 41 "beam/erl_term.h"
#    define CHECK_POINTER_MASK 0x0UL
#    define COMPRESS_POINTER(AnUint) (AnUint)
#    define EXPAND_POINTER(APointer) (APointer)
#  endif
# 45 "beam/erl_term.h"
#else
# 46 "beam/erl_term.h"
#  define HEAP_ON_C_STACK 1
#  define CHECK_POINTER_MASK 0x0UL
#  define COMPRESS_POINTER(AnUint) (AnUint)
#  define EXPAND_POINTER(APointer) (APointer)
#endif
# 51 "beam/erl_term.h"

struct erl_node_; /* Declared in erl_node_tables.h */

/*
 * Defining ET_DEBUG to 1 causes all type-specific data access
 * macros to perform runtime type checking. This is very useful
 * during development but reduces performance, so ET_DEBUG should
 * be disabled during benchmarking or release.
 */
/* #define ET_DEBUG 1 */
#ifndef ET_DEBUG
#  ifdef DEBUG
#    define ET_DEBUG 1
#  else
# 65 "beam/erl_term.h"
#    define ET_DEBUG 0
#  endif
# 67 "beam/erl_term.h"
#endif
# 68 "beam/erl_term.h"

#if ET_DEBUG
#define _ET_DECLARE_CHECKED(TF,F,TX) extern TF checked_##F(TX,const char*,unsigned);
#define _ET_APPLY(F,X)	checked_##F(X,__FILE__,__LINE__)
#else
# 73 "beam/erl_term.h"
#define _ET_DECLARE_CHECKED(TF,F,TX)
#define _ET_APPLY(F,X)	_unchecked_##F(X)
#endif
# 76 "beam/erl_term.h"

#define _TAG_PRIMARY_SIZE	2
#define _TAG_PRIMARY_MASK	0x3
#define TAG_PRIMARY_HEADER	0x0
#define TAG_PRIMARY_LIST	0x1
#define TAG_PRIMARY_BOXED	0x2
#define TAG_PRIMARY_IMMED1	0x3

#define primary_tag(x)	((x) & _TAG_PRIMARY_MASK)

#define _TAG_IMMED1_SIZE	4
#define _TAG_IMMED1_MASK	0xF
#define _TAG_IMMED1_PID		((0x0 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_IMMED1)
#define _TAG_IMMED1_PORT	((0x1 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_IMMED1)
#define _TAG_IMMED1_IMMED2	((0x2 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_IMMED1)
#define _TAG_IMMED1_SMALL	((0x3 << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_IMMED1)

#define _TAG_IMMED2_SIZE	6
#define _TAG_IMMED2_MASK	0x3F
#define _TAG_IMMED2_ATOM	((0x0 << _TAG_IMMED1_SIZE) | _TAG_IMMED1_IMMED2)
#define _TAG_IMMED2_CATCH	((0x1 << _TAG_IMMED1_SIZE) | _TAG_IMMED1_IMMED2)
#define _TAG_IMMED2_NIL		((0x3 << _TAG_IMMED1_SIZE) | _TAG_IMMED1_IMMED2)

/*
 * HEADER representation:
 *
 *	aaaaaaaaaaaaaaaaaaaaaaaaaatttt00	arity:26, tag:4
 *
 * HEADER tags:
 *
 *	0000	ARITYVAL
 *      0001    BINARY_AGGREGATE                |
 *	001x	BIGNUM with sign bit		|
 *	0100	REF				|
 *	0101	FUN				| THINGS
 *	0110	FLONUM				|
 *      0111    EXPORT                          |
 *	1000	REFC_BINARY	|		|
 *	1001	HEAP_BINARY	| BINARIES	|
 *	1010	SUB_BINARY	|		|
 *      1011    Not used; see comment below
 *      1100    EXTERNAL_PID  |                 |
 *      1101    EXTERNAL_PORT | EXTERNAL THINGS |
 *      1110    EXTERNAL_REF  |                 |
 *      1111    MAP
 *
 * COMMENTS:
 *
 * - The tag is zero for arityval and non-zero for thing headers.
 * - A single bit differentiates between positive and negative bignums.
 * - If more tags are needed, the REF and and EXTERNAL_REF tags could probably
 *   be combined to one tag.
 *
 * XXX: globally replace XXX_SUBTAG with TAG_HEADER_XXX
 */
#define ARITYVAL_SUBTAG		(0x0 << _TAG_PRIMARY_SIZE) /* TUPLE */
#define BIN_MATCHSTATE_SUBTAG	(0x1 << _TAG_PRIMARY_SIZE) 
#define POS_BIG_SUBTAG		(0x2 << _TAG_PRIMARY_SIZE) /* BIG: tags 2&3 */
#define NEG_BIG_SUBTAG		(0x3 << _TAG_PRIMARY_SIZE) /* BIG: tags 2&3 */
#define _BIG_SIGN_BIT		(0x1 << _TAG_PRIMARY_SIZE)
#define REF_SUBTAG		(0x4 << _TAG_PRIMARY_SIZE) /* REF */
#define FUN_SUBTAG		(0x5 << _TAG_PRIMARY_SIZE) /* FUN */
#define FLOAT_SUBTAG		(0x6 << _TAG_PRIMARY_SIZE) /* FLOAT */
#define EXPORT_SUBTAG		(0x7 << _TAG_PRIMARY_SIZE) /* FLOAT */
#define _BINARY_XXX_MASK	(0x3 << _TAG_PRIMARY_SIZE)
#define REFC_BINARY_SUBTAG	(0x8 << _TAG_PRIMARY_SIZE) /* BINARY */
#define HEAP_BINARY_SUBTAG	(0x9 << _TAG_PRIMARY_SIZE) /* BINARY */
#define SUB_BINARY_SUBTAG	(0xA << _TAG_PRIMARY_SIZE) /* BINARY */
/*   _BINARY_XXX_MASK depends on 0xB being unused */
#define EXTERNAL_PID_SUBTAG	(0xC << _TAG_PRIMARY_SIZE) /* EXTERNAL_PID */
#define EXTERNAL_PORT_SUBTAG	(0xD << _TAG_PRIMARY_SIZE) /* EXTERNAL_PORT */
#define EXTERNAL_REF_SUBTAG	(0xE << _TAG_PRIMARY_SIZE) /* EXTERNAL_REF */
#define MAP_SUBTAG		(0xF << _TAG_PRIMARY_SIZE) /* MAP */


#define _TAG_HEADER_ARITYVAL       (TAG_PRIMARY_HEADER|ARITYVAL_SUBTAG)
#define _TAG_HEADER_FUN	           (TAG_PRIMARY_HEADER|FUN_SUBTAG)
#define _TAG_HEADER_POS_BIG        (TAG_PRIMARY_HEADER|POS_BIG_SUBTAG)
#define _TAG_HEADER_NEG_BIG        (TAG_PRIMARY_HEADER|NEG_BIG_SUBTAG)
#define _TAG_HEADER_FLOAT          (TAG_PRIMARY_HEADER|FLOAT_SUBTAG)
#define _TAG_HEADER_EXPORT         (TAG_PRIMARY_HEADER|EXPORT_SUBTAG)
#define _TAG_HEADER_REF            (TAG_PRIMARY_HEADER|REF_SUBTAG)
#define _TAG_HEADER_REFC_BIN       (TAG_PRIMARY_HEADER|REFC_BINARY_SUBTAG)
#define _TAG_HEADER_HEAP_BIN       (TAG_PRIMARY_HEADER|HEAP_BINARY_SUBTAG)
#define _TAG_HEADER_SUB_BIN        (TAG_PRIMARY_HEADER|SUB_BINARY_SUBTAG)
#define _TAG_HEADER_EXTERNAL_PID   (TAG_PRIMARY_HEADER|EXTERNAL_PID_SUBTAG)
#define _TAG_HEADER_EXTERNAL_PORT  (TAG_PRIMARY_HEADER|EXTERNAL_PORT_SUBTAG)
#define _TAG_HEADER_EXTERNAL_REF   (TAG_PRIMARY_HEADER|EXTERNAL_REF_SUBTAG)
#define _TAG_HEADER_BIN_MATCHSTATE (TAG_PRIMARY_HEADER|BIN_MATCHSTATE_SUBTAG)
#define _TAG_HEADER_MAP	           (TAG_PRIMARY_HEADER|MAP_SUBTAG)


#define _TAG_HEADER_MASK	0x3F
#define _HEADER_SUBTAG_MASK	0x3C	/* 4 bits for subtag */
#define _HEADER_ARITY_OFFS	6

#define header_is_transparent(x) \
 (((x) & (_HEADER_SUBTAG_MASK)) == ARITYVAL_SUBTAG)
#define header_is_arityval(x)	(((x) & _HEADER_SUBTAG_MASK) == ARITYVAL_SUBTAG)
#define header_is_thing(x)	(!header_is_transparent((x)))
#define header_is_bin_matchstate(x)	((((x) & (_HEADER_SUBTAG_MASK)) == BIN_MATCHSTATE_SUBTAG))

#define _CPMASK		0x3

/* immediate object access methods */
#define is_immed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_IMMED1)
#define is_not_immed(x)		(!is_immed((x)))
#define IS_CONST(x)		is_immed((x))
#if TAG_PRIMARY_IMMED1 == _TAG_PRIMARY_MASK
#define is_both_immed(x,y)	is_immed(((x)&(y)))
#else
# 187 "beam/erl_term.h"
#define is_both_immed(x,y)	(is_immed((x)) && is_immed((y)))
#endif
# 189 "beam/erl_term.h"
#define is_not_both_immed(x,y)	(!is_both_immed((x),(y)))


/* boxed object access methods */
#if HALFWORD_HEAP
#define _is_taggable_pointer(x)	 (((UWord)(x) & (CHECK_POINTER_MASK | 0x3)) == 0)
#define _boxed_precond(x)        (is_boxed(x))
#else
# 197 "beam/erl_term.h"
#define _is_taggable_pointer(x)	 (((Uint)(x) & 0x3) == 0)
#define  _boxed_precond(x)       (is_boxed(x))
#endif
# 200 "beam/erl_term.h"
#define _is_aligned(x)		(((Uint)(x) & 0x3) == 0)
#define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED)
_ET_DECLARE_CHECKED(Eterm,make_boxed,const Eterm*)
#define make_boxed(x)		_ET_APPLY(make_boxed,(x))
#if 1
#define _is_not_boxed(x)	((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED))
#define _unchecked_is_boxed(x)	(!_is_not_boxed((x)))
_ET_DECLARE_CHECKED(int,is_boxed,Eterm)
#define is_boxed(x)		_ET_APPLY(is_boxed,(x))
#else
# 210 "beam/erl_term.h"
#define is_boxed(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED)
#endif
# 212 "beam/erl_term.h"
#define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED)))
_ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm)
#define boxed_val(x)		_ET_APPLY(boxed_val,(x))

/* cons cell ("list") access methods */
#define _unchecked_make_list(x)	((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST)
_ET_DECLARE_CHECKED(Eterm,make_list,const Eterm*)
#define make_list(x)		_ET_APPLY(make_list,(x))
#if 1
#define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST))
_ET_DECLARE_CHECKED(int,is_not_list,Eterm)
#define is_not_list(x)		_ET_APPLY(is_not_list,(x))
#define is_list(x)		(!is_not_list((x)))
#else
# 226 "beam/erl_term.h"
#define is_list(x)		(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_LIST)
#define is_not_list(x)		(!is_list((x)))
#endif
# 229 "beam/erl_term.h"
#if HALFWORD_HEAP
#define _list_precond(x)        (is_list(x))
#else
# 232 "beam/erl_term.h"
#define _list_precond(x)       (is_list(x))
#endif
# 234 "beam/erl_term.h"
#define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST))
_ET_DECLARE_CHECKED(Eterm*,list_val,Wterm)
#define list_val(x)		_ET_APPLY(list_val,(x))

#define CONS(hp, car, cdr) \
        (CAR(hp)=(car), CDR(hp)=(cdr), make_list(hp))

#define CAR(x)  ((x)[0])
#define CDR(x)  ((x)[1])

/* generic tagged pointer (boxed or list) access methods */
#define _unchecked_ptr_val(x)	((Eterm*) EXPAND_POINTER((x) & ~((Uint) 0x3)))
#define ptr_val(x)		_unchecked_ptr_val((x))	/*XXX*/
#define _unchecked_offset_ptr(x,offs)	((x)+((offs)*sizeof(Eterm)))
#define offset_ptr(x,offs)	_unchecked_offset_ptr(x,offs)	/*XXX*/
#define _unchecked_byte_offset_ptr(x,byte_offs)	((x)+(offs))
#define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs)  /*XXX*/

/* fixnum ("small") access methods */
#if defined(ARCH_64) && !HALFWORD_HEAP
#define SMALL_BITS	(64-4)
#define SMALL_DIGITS	(17)
#else
# 257 "beam/erl_term.h"
#define SMALL_BITS	(28)
#define SMALL_DIGITS	(8)
#endif
# 260 "beam/erl_term.h"
#define MAX_SMALL	((SWORD_CONSTANT(1) << (SMALL_BITS-1))-1)
#define MIN_SMALL	(-(SWORD_CONSTANT(1) << (SMALL_BITS-1)))
#define make_small(x)	(((Uint)(x) << _TAG_IMMED1_SIZE) + _TAG_IMMED1_SMALL)
#define is_small(x)	(((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_SMALL)
#define is_not_small(x)	(!is_small((x)))
#define is_byte(x)	(((x) & ((~(Uint)0 << (_TAG_IMMED1_SIZE+8)) + _TAG_IMMED1_MASK)) == _TAG_IMMED1_SMALL)
#define is_valid_bit_size(x) (((Sint)(x)) >= 0 && ((x) & 0x7F) == _TAG_IMMED1_SMALL)
#define is_not_valid_bit_size(x) (!is_valid_bit_size((x)))
#define MY_IS_SSMALL(x) (((Uint) ((((x)) >> (SMALL_BITS-1)) + 1)) < 2)
#define _unchecked_unsigned_val(x)	((x) >> _TAG_IMMED1_SIZE)
_ET_DECLARE_CHECKED(Uint,unsigned_val,Eterm)
#define unsigned_val(x)	_ET_APPLY(unsigned_val,(x))
#define _unchecked_signed_val(x)	((Sint)(x) >> _TAG_IMMED1_SIZE)
_ET_DECLARE_CHECKED(Sint,signed_val,Eterm)
#define signed_val(x)	_ET_APPLY(signed_val,(x))

#if _TAG_IMMED1_SMALL == 0x0F
#define is_both_small(x,y) (((x) & (y) & _TAG_IMMED1_MASK) == _TAG_IMMED1_SMALL)
#elif _TAG_IMMED1_SMALL == 0x00
# 279 "beam/erl_term.h"
#define is_both_small(x,y) ((((x)|(y)) & _TAG_IMMED1_MASK) == _TAG_IMMED1_SMALL)
#else
# 281 "beam/erl_term.h"
#define is_both_small(x,y) (is_small(x) && is_small(y))
#endif
# 283 "beam/erl_term.h"

/* NIL access methods */
#define NIL		((~((Uint) 0) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_NIL)
#define is_nil(x)	((x) == NIL)
#define is_not_nil(x)	((x) != NIL)

#define MAX_ATOM_INDEX (~(~((Uint) 0) << (sizeof(Uint)*8 - _TAG_IMMED2_SIZE)))

/* atom access methods */
#define make_atom(x)  ((Eterm)(((x) << _TAG_IMMED2_SIZE) + _TAG_IMMED2_ATOM))
#define is_atom(x)	(((x) & _TAG_IMMED2_MASK) == _TAG_IMMED2_ATOM)
#define is_not_atom(x)	(!is_atom(x))
#define _unchecked_atom_val(x)	((x) >> _TAG_IMMED2_SIZE)
_ET_DECLARE_CHECKED(Uint,atom_val,Eterm)
#define atom_val(x)	_ET_APPLY(atom_val,(x))

/* header (arityval or thing) access methods */
#define _make_header(sz,tag) ((Uint)(((Uint)(sz) << _HEADER_ARITY_OFFS) + (tag)))
#define is_header(x)	(((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_HEADER)
#define _unchecked_header_arity(x) \
    (is_map_header(x) ? MAP_HEADER_ARITY(x) : ((x) >> _HEADER_ARITY_OFFS))
_ET_DECLARE_CHECKED(Uint,header_arity,Eterm)
#define header_arity(x)	_ET_APPLY(header_arity,(x))

/* arityval access methods */
/* Erlang Spec. 4.7.3 defines max arity to 65535
 * we will however enforce max arity of 16777215 (24 bits)
 * (checked in bifs and asserted in debug)
 */
#define MAX_ARITYVAL            ((((Uint)1) << 24) - 1)
#define ERTS_MAX_TUPLE_SIZE     MAX_ARITYVAL

#define make_arityval(sz)	_make_header((sz),_TAG_HEADER_ARITYVAL)
#define is_arity_value(x)	(((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL)
#define is_sane_arity_value(x)	((((x) & _TAG_HEADER_MASK) == _TAG_HEADER_ARITYVAL) && \
				 (((x) >> _HEADER_ARITY_OFFS) <= MAX_ARITYVAL))
#define is_not_arity_value(x)	(!is_arity_value((x)))
#define _unchecked_arityval(x)	_unchecked_header_arity((x))
_ET_DECLARE_CHECKED(Uint,arityval,Eterm)
#define arityval(x)		_ET_APPLY(arityval,(x))

/* thing access methods */
#define is_thing(x)	(is_header((x)) && header_is_thing((x)))
#define is_thing_ptr(t) (is_thing((t)->header))
#define _unchecked_thing_arityval(x)	_unchecked_header_arity((x))
_ET_DECLARE_CHECKED(Uint,thing_arityval,Eterm)
#define thing_arityval(x)	_ET_APPLY(thing_arityval,(x))
#define _unchecked_thing_subtag(x)	((x) & _HEADER_SUBTAG_MASK)
_ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm)
#define thing_subtag(x)		_ET_APPLY(thing_subtag,(x))

/*
 * Magic non-value object.
 * Used as function return error and "absent value" indicator
 * in the original runtime system. The new runtime system also
 * uses it as forwarding marker for CONS cells.
 *
 * This value is 0 in the original runtime system, which unfortunately
 * promotes sloppy programming practices. It also prevents some useful
 * tag assignment schemes, e.g. using a 2-bit tag 00 for FIXNUM.
 *
 * To help find code which makes unwarranted assumptions about zero,
 * we now use a non-zero bit-pattern in debug mode.
 */
#if ET_DEBUG
# ifdef HIPE
   /* A very large (or negative) value as work-around for ugly hipe-bifs
      that return untagged integers (eg hipe_bs_put_utf8) */
#  define THE_NON_VALUE	_make_header((Uint)~0,_TAG_HEADER_FLOAT)
# else
# 353 "beam/erl_term.h"
#  define THE_NON_VALUE	_make_header(0,_TAG_HEADER_FLOAT)
# endif
# 355 "beam/erl_term.h"
#else
# 356 "beam/erl_term.h"
#define THE_NON_VALUE	(0)
#endif
# 358 "beam/erl_term.h"
#define is_non_value(x)	((x) == THE_NON_VALUE)
#define is_value(x)	((x) != THE_NON_VALUE)

/* binary object access methods */
#define is_binary_header(x) \
	((((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_REFC_BIN) || \
	 (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_HEAP_BIN) || \
	 (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_SUB_BIN))

#define make_binary(x)	make_boxed((Eterm*)(x))
#define is_binary(x)	(is_boxed((x)) && is_binary_header(*boxed_val((x))))
#define is_not_binary(x) (!is_binary((x)))
#define _unchecked_binary_val(x) _unchecked_boxed_val((x))
_ET_DECLARE_CHECKED(Eterm*,binary_val,Wterm)
#define binary_val(x)	_ET_APPLY(binary_val,(x))

/* process binaries stuff (special case of binaries) */
#define HEADER_PROC_BIN	_make_header(PROC_BIN_SIZE-1,_TAG_HEADER_REFC_BIN)

/* fun & export objects */
#define is_any_fun(x)   (is_fun((x)) || is_export((x)))
#define is_not_any_fun(x) (!is_any_fun((x)))

/* fun objects */
#define HEADER_FUN		_make_header(ERL_FUN_SIZE-2,_TAG_HEADER_FUN)
#define is_fun_header(x)	((x) == HEADER_FUN)
#define make_fun(x)		make_boxed((Eterm*)(x))
#define is_fun(x)		(is_boxed((x)) && is_fun_header(*boxed_val((x))))
#define is_not_fun(x)		(!is_fun((x)))
#define _unchecked_fun_val(x)   _unchecked_boxed_val((x))
_ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm)
#define fun_val(x)		_ET_APPLY(fun_val,(x))

/* export access methods */
#define make_export(x)	 make_boxed((x))
#define is_export(x)     (is_boxed((x)) && is_export_header(*boxed_val((x))))
#define is_not_export(x) (!is_export((x)))
#define _unchecked_export_val(x)   _unchecked_boxed_val(x)
_ET_DECLARE_CHECKED(Eterm*,export_val,Wterm)
#define export_val(x)	_ET_APPLY(export_val,(x))
#define is_export_header(x)	((x) == HEADER_EXPORT)
#if HALFWORD_HEAP
#define HEADER_EXPORT   _make_header(2,_TAG_HEADER_EXPORT)
#else
# 402 "beam/erl_term.h"
#define HEADER_EXPORT   _make_header(1,_TAG_HEADER_EXPORT)
#endif
# 404 "beam/erl_term.h"

/* bignum access methods */
#define make_pos_bignum_header(sz)	_make_header((sz),_TAG_HEADER_POS_BIG)
#define make_neg_bignum_header(sz)	_make_header((sz),_TAG_HEADER_NEG_BIG)
#define _is_bignum_header(x)	(((x) & (_TAG_HEADER_MASK-_BIG_SIGN_BIT)) == _TAG_HEADER_POS_BIG)
#define _unchecked_bignum_header_is_neg(x)	((x) & _BIG_SIGN_BIT)
_ET_DECLARE_CHECKED(int,bignum_header_is_neg,Eterm)
#define bignum_header_is_neg(x)	_ET_APPLY(bignum_header_is_neg,(x))
#define _unchecked_bignum_header_neg(x)	((x) | _BIG_SIGN_BIT)
_ET_DECLARE_CHECKED(Eterm,bignum_header_neg,Eterm)
#define bignum_header_neg(x)	_ET_APPLY(bignum_header_neg,(x))
#define _unchecked_bignum_header_arity(x)	_unchecked_header_arity((x))
_ET_DECLARE_CHECKED(Uint,bignum_header_arity,Eterm)
#define bignum_header_arity(x)	_ET_APPLY(bignum_header_arity,(x))
#define BIG_ARITY_MAX		((1 << 19)-1)
#define make_big(x)	make_boxed((x))
#define is_big(x)	(is_boxed((x)) && _is_bignum_header(*boxed_val((x))))
#define is_not_big(x)	(!is_big((x)))
#define _unchecked_big_val(x)	_unchecked_boxed_val((x))
_ET_DECLARE_CHECKED(Eterm*,big_val,Wterm)
#define big_val(x)		_ET_APPLY(big_val,(x))

/* flonum ("float") access methods */
#if defined(ARCH_64) && !HALFWORD_HEAP
#define HEADER_FLONUM   _make_header(1,_TAG_HEADER_FLOAT)
#else
# 430 "beam/erl_term.h"
#define HEADER_FLONUM	_make_header(2,_TAG_HEADER_FLOAT)
#endif
# 432 "beam/erl_term.h"
#define make_float(x)	make_boxed((x))
#define is_float(x)	(is_boxed((x)) && *boxed_val((x)) == HEADER_FLONUM)
#define is_not_float(x)	(!is_float(x))
#define _unchecked_float_val(x)	_unchecked_boxed_val((x))
_ET_DECLARE_CHECKED(Eterm*,float_val,Wterm)
#define float_val(x)	_ET_APPLY(float_val,(x))

/* Float definition for byte and word access */
typedef double ieee754_8;

typedef union float_def
{
    ieee754_8 fd;
    byte   fb[sizeof(ieee754_8)];
    Uint16 fs[sizeof(ieee754_8) / sizeof(Uint16)];
    Uint32 fw[sizeof(ieee754_8) / sizeof(Uint32)];
#if defined(ARCH_64) && !HALFWORD_HEAP
    Uint   fdw;
#endif
# 451 "beam/erl_term.h"
} FloatDef;

#if defined(ARCH_64) && !HALFWORD_HEAP

#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1)

#define PUT_DOUBLE(f, x)  *(x) = HEADER_FLONUM, \
                          *((x)+1) = (f).fdw
#define GET_DOUBLE_DATA(p, f) (f).fdw = *((Uint *) (p))
#define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fdw
#else
# 462 "beam/erl_term.h"
#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fw[0] = *((fval)+1), \
				      (f).fw[1] = *((fval)+2)

#define PUT_DOUBLE(f, x)  *(x) = HEADER_FLONUM, \
                          *((x)+1) = (f).fw[0], \
			  *((x)+2) = (f).fw[1]
#define GET_DOUBLE_DATA(p, f) (f).fw[0] = *((Uint *) (p)),\
                              (f).fw[1] = *(((Uint *) (p))+1)
#define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fw[0],\
                             *(((Uint *) (p))+1) = (f).fw[1]
#endif
# 473 "beam/erl_term.h"

#define GET_DOUBLE(x, f) FLOAT_VAL_GET_DOUBLE(float_val(x), f)

#define DOUBLE_DATA_WORDS (sizeof(ieee754_8)/sizeof(Eterm))
#define FLOAT_SIZE_OBJECT (DOUBLE_DATA_WORDS+1)

/* tuple access methods */
#define make_tuple(x)	make_boxed((x))
#define is_tuple(x)	(is_boxed((x)) && is_arity_value(*boxed_val((x))))
#define is_not_tuple(x)	(!is_tuple((x)))
#define is_tuple_arity(x, a) \
   (is_boxed((x)) && *boxed_val((x)) == make_arityval((a)))
#define is_not_tuple_arity(x, a) (!is_tuple_arity((x),(a)))
#define _unchecked_tuple_val(x)	_unchecked_boxed_val(x)
_ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm)
#define tuple_val(x)	_ET_APPLY(tuple_val,(x))

#define TUPLE0(t) \
        ((t)[0] = make_arityval(0), \
        make_tuple(t))
#define TUPLE1(t,e1) \
        ((t)[0] = make_arityval(1), \
        (t)[1] = (e1), \
        make_tuple(t))
#define TUPLE2(t,e1,e2) \
        ((t)[0] = make_arityval(2), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        make_tuple(t))
#define TUPLE3(t,e1,e2,e3) \
        ((t)[0] = make_arityval(3), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        (t)[3] = (e3), \
        make_tuple(t))
#define TUPLE4(t,e1,e2,e3,e4) \
        ((t)[0] = make_arityval(4), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        (t)[3] = (e3), \
        (t)[4] = (e4), \
        make_tuple(t))
#define TUPLE5(t,e1,e2,e3,e4,e5) \
        ((t)[0] = make_arityval(5), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        (t)[3] = (e3), \
        (t)[4] = (e4), \
        (t)[5] = (e5), \
        make_tuple(t))
#define TUPLE6(t,e1,e2,e3,e4,e5,e6) \
        ((t)[0] = make_arityval(6), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        (t)[3] = (e3), \
        (t)[4] = (e4), \
        (t)[5] = (e5), \
	(t)[6] = (e6), \
        make_tuple(t))

#define TUPLE7(t,e1,e2,e3,e4,e5,e6,e7) \
        ((t)[0] = make_arityval(7), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        (t)[3] = (e3), \
        (t)[4] = (e4), \
        (t)[5] = (e5), \
	(t)[6] = (e6), \
	(t)[7] = (e7), \
        make_tuple(t))

#define TUPLE8(t,e1,e2,e3,e4,e5,e6,e7,e8) \
        ((t)[0] = make_arityval(8), \
        (t)[1] = (e1), \
        (t)[2] = (e2), \
        (t)[3] = (e3), \
        (t)[4] = (e4), \
        (t)[5] = (e5), \
	(t)[6] = (e6), \
	(t)[7] = (e7), \
	(t)[8] = (e8), \
        make_tuple(t))

/* This macro get Size bits starting at low order position Pos
   and adjusts the bits to the right 
   bits are numbered from 0 - (sizeof(Uint)*8-1) */

#define _GETBITS(X,Pos,Size) (((X) >> (Pos)) & ~(~((Uint) 0) << (Size)))

/*
 * Creation in node specific data (pids, ports, refs)
 */

#define _CRE_SIZE  		2

/* MAX value for the creation field in pid, port and reference */
#define MAX_CREATION	(1 << _CRE_SIZE)

/*
 *  PID layout (internal pids):
 *
 *   |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *   |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *   |               |               |               |               |
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *   |n n n n n n n n n n n n n n n n n n n n n n n n n n n n|0 0|1 1|
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *  n : number
 *
 *  Old pid layout:
 *
 *   |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *   |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *   |               |               |               |               |
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *   |s s s|n n n n n n n n n n n n n n n|N N N N N N N N|c c|0 0|1 1|
 *   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *  s : serial
 *  n : number
 *  c : creation
 *  N : node number
 *
 */

#define _PID_SER_SIZE		(_PID_DATA_SIZE - _PID_NUM_SIZE)
#define _PID_NUM_SIZE 		15

#define _PID_DATA_SIZE		28
#define _PID_DATA_SHIFT		(_TAG_IMMED1_SIZE)

#define _GET_PID_DATA(X)	_GETBITS((X),_PID_DATA_SHIFT,_PID_DATA_SIZE)
#define _GET_PID_NUM(X)		_GETBITS((X),0,_PID_NUM_SIZE)
#define _GET_PID_SER(X)		_GETBITS((X),_PID_NUM_SIZE,_PID_SER_SIZE)

#define make_pid_data(Ser, Num) \
  ((Uint) ((Ser) << _PID_NUM_SIZE | (Num)))

#define is_internal_pid(x)	(((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_PID)
#define is_not_internal_pid(x)	(!is_internal_pid((x)))

#define _unchecked_internal_pid_node(x) erts_this_node
_ET_DECLARE_CHECKED(struct erl_node_*,internal_pid_node,Eterm)
#define internal_pid_node(x) _ET_APPLY(internal_pid_node,(x))

#define internal_pid_data_words(x) (1)

/* 
 *  PORT layout (internal ports):
 *
 *  |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *  |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *  |               |               |               |               |
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |n n n n n n n n n n n n n n n n n n n n n n n n n n n n|0 1|1 1|
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *  n : number
 *
 * Old port layout:
 *
 *  |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *  |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *  |               |               |               |               |
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |N N N N N N N N|n n n n n n n n n n n n n n n n n n|c c|0 1|1 1|
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *  s : serial
 *  n : number
 *  c : creation
 *  N : node number
 *
 */
#define _PORT_NUM_SIZE		_PORT_DATA_SIZE

#define _PORT_DATA_SIZE		28
#define _PORT_DATA_SHIFT	(_TAG_IMMED1_SIZE)

#define _GET_PORT_DATA(X)	_GETBITS((X),_PORT_DATA_SHIFT,_PORT_DATA_SIZE)
#define _GET_PORT_NUM(X)	_GETBITS((X), 0, _PORT_NUM_SIZE)


#define is_internal_port(x)	(((x) & _TAG_IMMED1_MASK) == _TAG_IMMED1_PORT)
#define is_not_internal_port(x)	(!is_internal_port(x))

#define _unchecked_internal_port_node(x) erts_this_node
_ET_DECLARE_CHECKED(struct erl_node_*,internal_port_node,Eterm)
#define internal_port_node(x) _ET_APPLY(internal_port_node,(x))

#define internal_port_data_words(x) (1)
/*
 *  Ref layout (internal references):
 *
 *  |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *  |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *  |               |               |               |               |
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1|0 1 0 0|0 0| Thing
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |0 0 0 0 0 0 0 0 0 0 0 0 0 0|r r r r r r r r r r r r r r r r r r| Data 0
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r| Data 1
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r| Data 2
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *
 *  r : reference number
 *  c : creation
 *
 *
 * Old "heap ref" layout:
 *
 *
 *  |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *  |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *  |               |               |               |               |
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1|0 1 0 0|0 0| Thing
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |N N N N N N N N|0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0|c c|0 1 1 1| Head
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |0 0 0 0 0 0 0 0 0 0 0 0 0 0|r r r r r r r r r r r r r r r r r r| Word 0
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r| Word 1
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r r| Word 2
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *  r : reference number
 *  c : creation
 *  N : node index
 *
 * Old "one-word ref" layout:
 *
 *  |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *  |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *  |               |               |               |               |
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |N N N N N N N N|r r r r r r r r r r r r r r r r r r|c c|T T T T|
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *
 *  r : reference number
 *  c : creation
 *  N : node index
 *
 */
#define _REF_NUM_SIZE		18

/* Old maximum number of references in the system */
#define MAX_REFERENCE		(1 << _REF_NUM_SIZE)
#define REF_MASK		(~(~((Uint)0) << _REF_NUM_SIZE))
#define ERTS_MAX_REF_NUMBERS	3
#define ERTS_REF_NUMBERS	ERTS_MAX_REF_NUMBERS

#if defined(ARCH_64) && !HALFWORD_HEAP
#  define ERTS_REF_WORDS	(ERTS_REF_NUMBERS/2 + 1)
#  define ERTS_REF_32BIT_WORDS  (ERTS_REF_NUMBERS+1)
#else
# 734 "beam/erl_term.h"
#  define ERTS_REF_WORDS	ERTS_REF_NUMBERS
#  define ERTS_REF_32BIT_WORDS  ERTS_REF_NUMBERS
#endif
# 737 "beam/erl_term.h"

typedef struct {
    Eterm      header;
    union {
	Uint32 ui32[ERTS_REF_32BIT_WORDS];
	Uint   ui[ERTS_REF_WORDS];
    } data;
} RefThing;

#define REF_THING_SIZE (sizeof(RefThing)/sizeof(Uint))
#define REF_THING_HEAD_SIZE (sizeof(Eterm)/sizeof(Uint))

#define make_ref_thing_header(DW) \
  _make_header((DW)+REF_THING_HEAD_SIZE-1,_TAG_HEADER_REF)

#if defined(ARCH_64) && !HALFWORD_HEAP

/*
 * Ref layout on a 64-bit little endian machine:
 *
 * 63             31             0
 * +--------------+--------------+
 * |         Thing word          |
 * +--------------+--------------+
 * |  Data 0      | 32-bit arity |
 * +--------------+--------------+
 * |  Data 2      | Data 1       |
 * +--------------+--------------+
 *
 * Data is stored as an Uint32 array with 32-bit arity as first number.
 */

#define write_ref_thing(Hp, R0, R1, R2)					\
do {									\
  ((RefThing *) (Hp))->header  = make_ref_thing_header(ERTS_REF_WORDS);	\
  ((RefThing *) (Hp))->data.ui32[0] = ERTS_REF_NUMBERS;			\
  ((RefThing *) (Hp))->data.ui32[1] = (R0);				\
  ((RefThing *) (Hp))->data.ui32[2] = (R1);				\
  ((RefThing *) (Hp))->data.ui32[3] = (R2);				\
} while (0)

#else
# 779 "beam/erl_term.h"

#define write_ref_thing(Hp, R0, R1, R2)					\
do {									\
  ((RefThing *) (Hp))->header  = make_ref_thing_header(ERTS_REF_WORDS);	\
  ((RefThing *) (Hp))->data.ui32[0] = (R0);				\
  ((RefThing *) (Hp))->data.ui32[1] = (R1);				\
  ((RefThing *) (Hp))->data.ui32[2] = (R2);				\
} while (0)

#endif
# 789 "beam/erl_term.h"

#define is_ref_thing_header(x)	(((x) & _TAG_HEADER_MASK) == _TAG_HEADER_REF)
#define make_internal_ref(x)	make_boxed((Eterm*)(x))

#define _unchecked_ref_thing_ptr(x) \
  ((RefThing*) _unchecked_internal_ref_val(x))
#define ref_thing_ptr(x) \
  ((RefThing*) internal_ref_val(x))

#define is_internal_ref(x) \
    (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x))))

#define is_not_internal_ref(x) \
  (!is_internal_ref((x)))

#define _unchecked_internal_ref_val(x) _unchecked_boxed_val((x))
_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm)
#define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x))

#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t)))
#define _unchecked_internal_ref_data_words(x) \
 (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x)))
_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm)
#define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x))

#define internal_thing_ref_data(thing) ((thing)->data.ui32)
#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x)))
_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm)
#define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x))

#define _unchecked_internal_ref_node(x) erts_this_node
_ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm)
#define internal_ref_node(x) _ET_APPLY(internal_ref_node,(x))

/*
 *
 *  External thing layout (external pids, ports, and refs):
 *
 *  |3 3 2 2 2 2 2 2|2 2 2 2 1 1 1 1|1 1 1 1 1 1    |               |
 *  |1 0 9 8 7 6 5 4|3 2 1 0 9 8 7 6|5 4 3 2 1 0 9 8|7 6 5 4 3 2 1 0|
 *  |               |               |               |               |
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |A A A A A A A A A A A A A A A A A A A A A A A A A A|t t t t|0 0| Thing
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E| ErlNode
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  |X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X| Data 0
 *  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 *  .                                                               .   .
 *  .                                                               .   .
 *  .                                                               .   .
 *
 *  A : Arity
 *  t : External pid thing tag  (1100)
 *  t : External port thing tag (1101)
 *  t : External ref thing tag  (1110)
 *  N : Next (off_heap) pointer
 *  E : ErlNode pointer
 *  X : Type specific data
 *
 *  External pid and port layout:
 *    External pids and ports only have one data word (Data 0) which has
 *    the same layout as internal pids resp. internal ports.
 *
 *  External refs layout:
 *    External refs has the same layout for the data words as in the internal
 *    ref. 
 *
 */

/* XXX:PaN - this structure is not perfect for halfword heap, it takes
   a lot of memory due to padding, and the array will not begin at the end of the
   structure, as otherwise expected. Be sure to access data.ui32 array and not try
   to do pointer manipulation on an Eterm * to reach the actual data...

   XXX:Sverk - Problem made worse by "one off-heap list" when 'next' pointer
     must align with 'next' in ProcBin, erl_fun_thing and erl_off_heap_header.
*/
typedef struct external_thing_ {
    /*                                 ----+                        */
    Eterm                   header;     /* |                        */
    struct erl_node_*       node;       /*  > External thing head   */
    struct erl_off_heap_header* next;   /* |                        */
    /*                                 ----+                        */
    union {
	Uint32              ui32[1];
	Uint                ui[1];
    } data;
} ExternalThing;

#define EXTERNAL_THING_HEAD_SIZE (sizeof(ExternalThing)/sizeof(Uint) - 1)

#define make_external_pid_header(DW) \
  _make_header((DW)+EXTERNAL_THING_HEAD_SIZE-1,_TAG_HEADER_EXTERNAL_PID)
#define is_external_pid_header(x) \
  (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_EXTERNAL_PID)

#define make_external_port_header(DW) \
  _make_header((DW)+EXTERNAL_THING_HEAD_SIZE-1,_TAG_HEADER_EXTERNAL_PORT)
#define is_external_port_header(x) \
  (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_EXTERNAL_PORT)

#define make_external_ref_header(DW) \
  _make_header((DW)+EXTERNAL_THING_HEAD_SIZE-1,_TAG_HEADER_EXTERNAL_REF)
#define is_external_ref_header(x) \
  (((x) & _TAG_HEADER_MASK) == _TAG_HEADER_EXTERNAL_REF)

#define is_external_header(x) \
  (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID \
   && ((x) & _TAG_HEADER_MASK) != _TAG_HEADER_MAP)

#define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x))))

#define is_external_pid(x) \
  (is_boxed((x)) && is_external_pid_header(*boxed_val((x))))
#define is_external_port(x) \
    (is_boxed((x)) && is_external_port_header(*boxed_val((x))))

#define is_external_ref(x) (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x))))

#define _unchecked_is_external(x) \
  (_unchecked_is_boxed((x)) && is_external_header(*_unchecked_boxed_val((x))))

#define is_not_external(x)	(!is_external((x)))
#define is_not_external_pid(x)	(!is_external_pid((x)))
#define is_not_external_port(x)	(!is_external_port((x)))
#define is_not_external_ref(x)	(!is_external_ref((x)))


#define make_external(x)		make_boxed((Eterm *) (x))

#define make_external_pid		make_external
#define make_external_port		make_external
#define make_external_ref		make_external

#define _unchecked_external_val(x) _unchecked_boxed_val((x))
_ET_DECLARE_CHECKED(Eterm*,external_val,Wterm)
#define external_val(x) _ET_APPLY(external_val,(x))

#define external_thing_ptr(x) ((ExternalThing *) external_val((x)))
#define _unchecked_external_thing_ptr(x) \
  ((ExternalThing *) _unchecked_external_val((x)))

#define _unchecked_external_thing_data_words(thing) \
    (_unchecked_thing_arityval((thing)->header) + (1 - EXTERNAL_THING_HEAD_SIZE))
_ET_DECLARE_CHECKED(Uint,external_thing_data_words,ExternalThing*)
#define external_thing_data_words(thing) _ET_APPLY(external_thing_data_words,(thing))

#define _unchecked_external_data_words(x) \
    _unchecked_external_thing_data_words(_unchecked_external_thing_ptr((x)))
_ET_DECLARE_CHECKED(Uint,external_data_words,Wterm)
#define external_data_words(x) _ET_APPLY(external_data_words,(x))

#define _unchecked_external_data(x) (_unchecked_external_thing_ptr((x))->data.ui)
#define _unchecked_external_node(x) (_unchecked_external_thing_ptr((x))->node)

#define external_data(x) (external_thing_ptr((x))->data.ui)
#define external_node(x) (external_thing_ptr((x))->node)

#define _unchecked_external_pid_data_words(x) \
  _unchecked_external_data_words((x))
_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Wterm)
#define external_pid_data_words(x) _ET_APPLY(external_pid_data_words,(x))

#define _unchecked_external_pid_data(x) _unchecked_external_data((x))[0]
_ET_DECLARE_CHECKED(Uint,external_pid_data,Wterm)
#define external_pid_data(x) _ET_APPLY(external_pid_data,(x))

#define _unchecked_external_pid_node(x) _unchecked_external_node((x))
_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Wterm)
#define external_pid_node(x) _ET_APPLY(external_pid_node,(x))

#define external_pid_number(x) _GET_PID_NUM(external_pid_data((x)))
#define external_pid_serial(x) _GET_PID_SER(external_pid_data((x)))

#define _unchecked_external_port_data_words(x) \
  _unchecked_external_data_words((x))
_ET_DECLARE_CHECKED(Uint,external_port_data_words,Wterm)
#define external_port_data_words(x) _ET_APPLY(external_port_data_words,(x))

#define _unchecked_external_port_data(x) _unchecked_external_data((x))[0]
_ET_DECLARE_CHECKED(Uint,external_port_data,Wterm)
#define external_port_data(x) _ET_APPLY(external_port_data,(x))

#define _unchecked_external_port_node(x) _unchecked_external_node((x))
_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Wterm)
#define external_port_node(x) _ET_APPLY(external_port_node,(x))

#define external_port_number(x) _GET_PORT_NUM(external_port_data((x)))

#define _unchecked_external_ref_data_words(x) \
  _unchecked_external_data_words((x))
_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Wterm)
#define external_ref_data_words(x) _ET_APPLY(external_ref_data_words,(x))
#define external_thing_ref_data_words(thing) external_thing_data_words(thing)

#define _unchecked_external_ref_data(x) (_unchecked_external_thing_ptr((x))->data.ui32)
_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm)
#define external_ref_data(x) _ET_APPLY(external_ref_data,(x))
#define external_thing_ref_data(thing) ((thing)->data.ui32)

#define _unchecked_external_ref_node(x) _unchecked_external_node((x))
_ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm)
#define external_ref_node(x) _ET_APPLY(external_ref_node,(x))

/* maps */

#define MAP_HEADER_TAG_SZ                 (2)
#define MAP_HEADER_ARITY_SZ               (8)
#define MAP_HEADER_VAL_SZ                 (16)

#define MAP_HEADER_TAG_FLATMAP_HEAD       (0x0)
#define MAP_HEADER_TAG_HAMT_NODE_BITMAP   (0x1)
#define MAP_HEADER_TAG_HAMT_HEAD_ARRAY    (0x2)
#define MAP_HEADER_TAG_HAMT_HEAD_BITMAP   (0x3)

#define MAP_HEADER_TYPE(Hdr)  (((Hdr) >> (_HEADER_ARITY_OFFS)) & (0x3))
#define MAP_HEADER_ARITY(Hdr) (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ)) & (0xff))
#define MAP_HEADER_VAL(Hdr)   (((Hdr) >> (_HEADER_ARITY_OFFS + MAP_HEADER_TAG_SZ + MAP_HEADER_ARITY_SZ)) & (0xffff))

#define make_hashmap(x)		      make_boxed((Eterm*)(x))
#define make_hashmap_rel 	      make_boxed_rel
#define is_hashmap(x)		      (is_boxed((x)) && is_hashmap_header(*boxed_val((x))))
#define is_not_hashmap(x)             (!is_hashmap(x))
#define is_hashmap_rel(RTERM,BASE)    is_hashmap(rterm2wterm(RTERM,BASE))
#define is_hashmap_header(x)	      (((x) & (_HEADER_MAP_HASHMAP_HEAD_MASK)) == HAMT_SUBTAG_HEAD_ARRAY)
#define hashmap_val(x)		      _unchecked_boxed_val((x))
#define hashmap_val_rel(RTERM, BASE)  hashmap_val(rterm2wterm(RTERM, BASE))

#define make_flatmap(x)               make_boxed((Eterm*)(x))
#define make_flatmap_rel(x, BASE)     make_boxed_rel((Eterm*)(x),(BASE))
#define is_flatmap(x)                 (is_boxed((x)) && is_flatmap_header(*boxed_val((x))))
#define is_flatmap_rel(RTERM,BASE)    is_flatmap(rterm2wterm(RTERM,BASE))
#define is_not_flatmap(x)             (!is_flatmap((x)))
#define is_flatmap_header(x)          (((x) & (_HEADER_MAP_SUBTAG_MASK)) == HAMT_SUBTAG_HEAD_FLATMAP)
#define flatmap_val(x)                (_unchecked_boxed_val((x)))
#define flatmap_val_rel(RTERM, BASE)  flatmap_val(rterm2wterm(RTERM, BASE))

#define is_map_header(x)       (((x) & (_TAG_HEADER_MASK)) == _TAG_HEADER_MAP)
#define is_map(x)              (is_boxed((x)) && is_map_header(*boxed_val(x)))
#define is_not_map(x)          (!is_map(x))
#define is_map_rel(RTERM,BASE) is_map(rterm2wterm(RTERM,BASE))

/* number tests */

#define is_integer(x)		(is_small(x) || is_big(x))
#define is_not_integer(x)	(!is_integer(x))
#define is_number(x)		(is_integer(x) || is_float(x))

#define SMALL_MINUS_ONE	make_small(-1)
#define SMALL_ZERO	make_small(0)
#define SMALL_ONE	make_small(1)

#define ENULL		0

/* on some architectures CP contains labels which are not aligned */
#ifdef NOT_ALIGNED
#error "fix yer arch, like"
#endif
# 1050 "beam/erl_term.h"

#define _unchecked_make_cp(x)	((Eterm) COMPRESS_POINTER(x))
_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*)
#define make_cp(x)	_ET_APPLY(make_cp,(x))

#define is_not_CP(x)	((x) & _CPMASK)
#define is_CP(x)	(!is_not_CP(x))

#define _unchecked_cp_val(x)	((BeamInstr*) EXPAND_POINTER(x))
_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm)
#define cp_val(x)	_ET_APPLY(cp_val,(x))

#define make_catch(x)	(((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH)
#define is_catch(x)	(((x) & _TAG_IMMED2_MASK) == _TAG_IMMED2_CATCH)
#define is_not_catch(x)	(!is_catch(x))
#define _unchecked_catch_val(x)	((x) >> _TAG_IMMED2_SIZE)
_ET_DECLARE_CHECKED(Uint,catch_val,Eterm)
#define catch_val(x)	_ET_APPLY(catch_val,(x))

#define make_blank(X)	((X) = NIL)

/*
 * Overloaded tags.
 *
 * SMALL = 15
 * ATOM/NIL=7
 *
 * Note that the two least significant bits in SMALL/ATOM/NIL always are 3;
 * thus, we can distinguish register from literals by looking at only these
 * two bits.
 */

#define X_REG_DEF	0
#define Y_REG_DEF	1
#define R_REG_DEF	2

#define beam_reg_tag(x)	((x) & 3)

#define make_rreg()	R_REG_DEF
#define make_xreg(ix)	(((ix) * sizeof(Eterm)) | X_REG_DEF)
#define make_yreg(ix)	(((ix) * sizeof(Eterm)) | Y_REG_DEF)

#define _is_xreg(x)	(beam_reg_tag(x) == X_REG_DEF)
#define _is_yreg(x)	(beam_reg_tag(x) == Y_REG_DEF)

#define _unchecked_x_reg_offset(R)	((R) - X_REG_DEF)
_ET_DECLARE_CHECKED(Uint,x_reg_offset,Uint)
#define x_reg_offset(R)	_ET_APPLY(x_reg_offset,(R))

#define _unchecked_y_reg_offset(R)	((R) - Y_REG_DEF)
_ET_DECLARE_CHECKED(Uint,y_reg_offset,Uint)
#define y_reg_offset(R)	_ET_APPLY(y_reg_offset,(R))

#define reg_index(R) ((R) / sizeof(Eterm))

#define _unchecked_x_reg_index(R)	((R) >> 2)
_ET_DECLARE_CHECKED(Uint,x_reg_index,Uint)
#define x_reg_index(R)	_ET_APPLY(x_reg_index,(R))

#define _unchecked_y_reg_index(R)	((R) >> 2)
_ET_DECLARE_CHECKED(Uint,y_reg_index,Uint)
#define y_reg_index(R)	_ET_APPLY(y_reg_index,(R))

/*
 * Backwards compatibility definitions:
 * - #define virtual *_DEF constants with values that fit term order:
 *   number < atom < ref < fun < port < pid < tuple < map < nil < cons < binary
 * - tag_val_def() function generates virtual _DEF tag
 * - not_eq_tags() and NUMBER_CODE() defined in terms
 *   of the tag_val_def() function
 */

#define BINARY_DEF		0x0
#define LIST_DEF		0x1
#define NIL_DEF			0x2
#define MAP_DEF			0x3
#define TUPLE_DEF		0x4
#define PID_DEF			0x5
#define EXTERNAL_PID_DEF	0x6
#define PORT_DEF		0x7
#define EXTERNAL_PORT_DEF	0x8
#define EXPORT_DEF		0x9
#define FUN_DEF			0xa
#define REF_DEF			0xb
#define EXTERNAL_REF_DEF	0xc
#define ATOM_DEF		0xd
#define FLOAT_DEF		0xe
#define BIG_DEF			0xf
#define SMALL_DEF		0x10
#define MATCHSTATE_DEF          0x11   /* not a "real" term */

#define FIRST_VACANT_TAG_DEF    0x12

#if ET_DEBUG
extern unsigned tag_val_def_debug(Wterm, const char*, unsigned);
#define tag_val_def(x)	tag_val_def_debug((x),__FILE__,__LINE__)
#else
# 1147 "beam/erl_term.h"
extern unsigned tag_val_def(Wterm);
#endif
# 1149 "beam/erl_term.h"
#define not_eq_tags(X,Y)	(tag_val_def((X)) ^ tag_val_def((Y)))

#define NUMBER_CODE(x,y)	((tag_val_def(x) << 5) | tag_val_def(y))
#define _NUMBER_CODE(TX,TY)	((TX << 5) | TY)
#define SMALL_SMALL	_NUMBER_CODE(SMALL_DEF,SMALL_DEF)
#define SMALL_BIG 	_NUMBER_CODE(SMALL_DEF,BIG_DEF)
#define SMALL_FLOAT 	_NUMBER_CODE(SMALL_DEF,FLOAT_DEF)
#define BIG_SMALL 	_NUMBER_CODE(BIG_DEF,SMALL_DEF)
#define BIG_BIG 	_NUMBER_CODE(BIG_DEF,BIG_DEF)
#define BIG_FLOAT 	_NUMBER_CODE(BIG_DEF,FLOAT_DEF)
#define FLOAT_SMALL 	_NUMBER_CODE(FLOAT_DEF,SMALL_DEF)
#define FLOAT_BIG 	_NUMBER_CODE(FLOAT_DEF,BIG_DEF)
#define FLOAT_FLOAT	_NUMBER_CODE(FLOAT_DEF,FLOAT_DEF)

#if HALFWORD_HEAP
#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE)))
#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE))

#else /* HALFWORD_HEAP */
# 1168 "beam/erl_term.h"

#define ptr2rel(PTR,BASE) (PTR)
#define rterm2wterm(REL,BASE) (REL)

#endif /* !HALFWORD_HEAP */
# 1173 "beam/erl_term.h"

#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE))
#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE))
#define make_fun_rel make_boxed_rel
#define make_binary_rel make_boxed_rel
#define make_tuple_rel make_boxed_rel
#define make_external_rel make_boxed_rel
#define make_internal_ref_rel make_boxed_rel
#define make_big_rel make_boxed_rel

#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE))
#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE))
#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE))
#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE))
#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE))
#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE))
#define big_val_rel(RTERM,BASE)	big_val(rterm2wterm(RTERM,BASE))
#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE))
#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE))

#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE))
#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE))

#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE))
#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE))

#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE))
#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE))
#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE))

#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE))
#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE))
#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE))
#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE))
#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE))
#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE))

#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f)

#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE))
#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE))
#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE))
#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE))
#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE))

#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE))


#if HALFWORD_HEAP
ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base)
{
    /* If bases differ, assume a and b are on different "heaps",
       ie can only be same if immed */
    ASSERT(a_base == b_base || is_immed(a) || is_immed(b)
	   || rterm2wterm(a,a_base) != rterm2wterm(b,b_base));

    return a == b && (a_base == b_base || is_immed(a));
}
#endif
# 1235 "beam/erl_term.h"

#else /* !HALFWORD_HEAP */
# 1237 "beam/erl_term.h"
#define is_same(A,A_BASE,B,B_BASE) ((A)==(B))
#endif
# 1239 "beam/erl_term.h"

#endif	/* __ERL_TERM_H */
# 1241 "beam/erl_term.h"

# 269 "beam/erl_threads.h" 2

#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
/*
 * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
 * only seen this bug in glibc versions before 2.4. Note that condition
 * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
 * this bug since these implementations may use mutexes internally.
 */
#  define ERTS_THR_HAVE_BUSY_DESTROY_BUG
#endif
# 279 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__)
#define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__)
#endif
# 296 "beam/erl_threads.h"

#define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
typedef ethr_thr_opts erts_thr_opts_t;
typedef ethr_init_data erts_thr_init_data_t;
typedef ethr_late_init_data erts_thr_late_init_data_t;
typedef ethr_tid erts_tid_t;

/* mutex */
typedef struct {
    ethr_mutex mtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 309 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 312 "beam/erl_threads.h"

} erts_mtx_t;
typedef ethr_cond erts_cnd_t;

/* rwmutex */
typedef struct {
    ethr_rwmutex rwmtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 322 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 325 "beam/erl_threads.h"
} erts_rwmtx_t;

#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER
#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
typedef ethr_rwmutex_opt erts_rwmtx_opt_t;

typedef ethr_tsd_key erts_tsd_key_t;
typedef ethr_ts_event erts_tse_t;
#define erts_dw_aint_t ethr_dw_sint_t
#define erts_dw_atomic_t ethr_dw_atomic_t
#define erts_aint_t ethr_sint_t
#define erts_atomic_t ethr_atomic_t
#define erts_aint32_t ethr_sint32_t 
#define erts_atomic32_t ethr_atomic32_t

#if defined(ARCH_32)
#  define erts_atomic64_t ethr_dw_atomic_t
#  define erts_aint64_t ethr_sint64_t
#elif defined(ARCH_64)
# 352 "beam/erl_threads.h"
#  define erts_atomic64_t ethr_atomic_t
#  define erts_aint64_t ethr_sint_t
#else
# 355 "beam/erl_threads.h"
#  error "Not supported architecture"
#endif
# 357 "beam/erl_threads.h"

#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD

/* spinlock */
typedef struct {
    ethr_spinlock_t slck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 367 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 370 "beam/erl_threads.h"
} erts_spinlock_t;

/* rwlock */
typedef struct {
    ethr_rwlock_t rwlck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 378 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 381 "beam/erl_threads.h"
} erts_rwlock_t;

__decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
                                 /* implemented in erl_init.c */

#define ERTS_THR_INIT_DATA_DEF_INITER	ETHR_INIT_DATA_DEFAULT_INITER
#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
                                        ETHR_LATE_INIT_DATA_DEFAULT_INITER

#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
#  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
#endif
# 393 "beam/erl_threads.h"

#else /* #ifdef USE_THREADS */
# 395 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#define ERTS_THR_OPTS_DEFAULT_INITER 0
typedef int erts_thr_opts_t;
typedef int erts_thr_init_data_t;
typedef int erts_thr_late_init_data_t;
typedef int erts_tid_t;
typedef int erts_mtx_t;
typedef int erts_cnd_t;
#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_RWMTX_TYPE_NORMAL 0
#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_RWMTX_LONG_LIVED 0
#define ERTS_RWMTX_SHORT_LIVED 0
#define ERTS_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_rwmtx_opt_t;
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;

typedef struct { SWord sint[2]; } erts_dw_aint_t;
typedef SWord erts_aint_t;
typedef Sint32 erts_aint32_t;
typedef Sint64 erts_aint64_t;

#define erts_dw_atomic_t erts_dw_aint_t
#define erts_atomic_t erts_aint_t
#define erts_atomic32_t erts_aint32_t
#define erts_atomic64_t erts_aint64_t

#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
#else
# 439 "beam/erl_threads.h"
typedef struct { int gcc_is_buggy; } erts_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif
# 442 "beam/erl_threads.h"

#ifdef WORDS_BIGENDIAN
#define ERTS_DW_AINT_LOW_WORD 1
#define ERTS_DW_AINT_HIGH_WORD 0
#else
# 447 "beam/erl_threads.h"
#define ERTS_DW_AINT_LOW_WORD 0
#define ERTS_DW_AINT_HIGH_WORD 1
#endif
# 450 "beam/erl_threads.h"

#define ERTS_MTX_INITER			0
#define ERTS_CND_INITER			0
#define ERTS_THR_INIT_DATA_DEF_INITER	0

#define ERTS_HAVE_REC_MTX_INIT		1

#endif /* #ifdef USE_THREADS */
# 458 "beam/erl_threads.h"

#define erts_no_dw_atomic_t erts_dw_aint_t
#define erts_no_atomic_t erts_aint_t
#define erts_no_atomic32_t erts_aint32_t
#define erts_no_atomic64_t erts_aint64_t

#define ERTS_AINT_NULL ((erts_aint_t) NULL)

#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))

ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
				     void *arg, erts_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid);
ERTS_GLB_INLINE void erts_thr_exit(void *res);
ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
				     int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
					 Uint16 opt, int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
					    char *name,
					    Eterm extra,
					    int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file,
				       unsigned int line);
ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file,
				     unsigned int line);
#else
# 499 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx);
#endif
# 502 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
					   erts_rwmtx_opt_t *opt,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
				       char *name,
				       Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
					 erts_rwmtx_opt_t *opt,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 529 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
#endif
# 534 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);

ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *val,
					      erts_no_dw_atomic_t *old_val);
ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
						    erts_aint_t i);
ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
						    erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
						     erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
						erts_aint_t new);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
						   erts_aint_t new,
						   erts_aint_t expected);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var,
						     erts_aint_t mask,
						     erts_aint_t set);
ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
							erts_aint32_t i);
ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
							erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
							 erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
						    erts_aint32_t new);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
						       erts_aint32_t new,
						       erts_aint32_t expected);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
							 erts_aint32_t mask,
							 erts_aint32_t set);
ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
							erts_aint64_t i);
ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
							erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
							 erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
						    erts_aint64_t new);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
						       erts_aint64_t new,
						       erts_aint64_t expected);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
							 erts_aint64_t mask,
							 erts_aint64_t set);

ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
					      char *name,
					      Eterm extra,
					      Uint16 opt);
ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
					  char *name,
					  Eterm extra);
ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
					char *name);
ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line);
#else
# 626 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
#endif
# 628 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_rwlock_init_x(erts_rwlock_t *lock,
					char *name,
					Eterm extra);
ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
				      char *name);
ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
#else
# 640 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_read_lock(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
#endif
# 643 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname);
ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
ERTS_GLB_INLINE int erts_thr_get_main_status(void);
ERTS_GLB_INLINE void erts_thr_yield(void);


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS
#define ERTS_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
				      sigset_t *oset);
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);

#ifdef USE_THREADS
ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
#endif
# 674 "beam/erl_threads.h"

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 676 "beam/erl_threads.h"

#ifdef USE_THREADS

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \
do {									\
    Type act = ReadOp((VarP));						\
    while (1) {								\
	Type exp = act;							\
	Type new = exp & ~(Mask);					\
	new |= ((Mask) & (Set));					\
	act = CmpxchgOp((VarP), new, exp);				\
	if (act == exp)							\
	    return act;							\
    }									\
} while (0)
#endif
# 750 "beam/erl_threads.h"

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var);
ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var);

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of this file for info on atomics.
 */

/* Double word size atomics */

#define erts_dw_atomic_init_nob ethr_dw_atomic_init
#define erts_dw_atomic_set_nob ethr_dw_atomic_set
#define erts_dw_atomic_read_nob ethr_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb

#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb

#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb

#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb

#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb

#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    sint[0] = val->sint[0];
    sint[1] = val->sint[1];    
}

ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    val->sint[0] = sint[0];
    val->sint[1] = sint[1];
}

#endif
# 825 "beam/erl_threads.h"

/* Word size atomics */

#define erts_atomic_init_nob ethr_atomic_init
#define erts_atomic_set_nob ethr_atomic_set
#define erts_atomic_read_nob ethr_atomic_read
#define erts_atomic_inc_read_nob ethr_atomic_inc_read
#define erts_atomic_dec_read_nob ethr_atomic_dec_read
#define erts_atomic_inc_nob ethr_atomic_inc
#define erts_atomic_dec_nob ethr_atomic_dec
#define erts_atomic_add_read_nob ethr_atomic_add_read
#define erts_atomic_add_nob ethr_atomic_add
#define erts_atomic_read_bor_nob ethr_atomic_read_bor
#define erts_atomic_read_band_nob ethr_atomic_read_band
#define erts_atomic_xchg_nob ethr_atomic_xchg
#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg,
			    var, mask, set);
}
#endif
# 854 "beam/erl_threads.h"

#define erts_atomic_init_mb ethr_atomic_init_mb
#define erts_atomic_set_mb ethr_atomic_set_mb
#define erts_atomic_read_mb ethr_atomic_read_mb
#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic_inc_mb ethr_atomic_inc_mb
#define erts_atomic_dec_mb ethr_atomic_dec_mb
#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic_add_mb ethr_atomic_add_mb
#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 881 "beam/erl_threads.h"

#define erts_atomic_init_acqb ethr_atomic_init_acqb
#define erts_atomic_set_acqb ethr_atomic_set_acqb
#define erts_atomic_read_acqb ethr_atomic_read_acqb
#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic_add_acqb ethr_atomic_add_acqb
#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 908 "beam/erl_threads.h"

#define erts_atomic_init_relb ethr_atomic_init_relb
#define erts_atomic_set_relb ethr_atomic_set_relb
#define erts_atomic_read_relb ethr_atomic_read_relb
#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic_inc_relb ethr_atomic_inc_relb
#define erts_atomic_dec_relb ethr_atomic_dec_relb
#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic_add_relb ethr_atomic_add_relb
#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 935 "beam/erl_threads.h"

#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 962 "beam/erl_threads.h"

#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic_inc_rb ethr_atomic_inc_rb
#define erts_atomic_dec_rb ethr_atomic_dec_rb
#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic_add_rb ethr_atomic_add_rb
#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 989 "beam/erl_threads.h"

#define erts_atomic_init_wb ethr_atomic_init_wb
#define erts_atomic_set_wb ethr_atomic_set_wb
#define erts_atomic_read_wb ethr_atomic_read_wb
#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic_inc_wb ethr_atomic_inc_wb
#define erts_atomic_dec_wb ethr_atomic_dec_wb
#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic_add_wb ethr_atomic_add_wb
#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1018 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    return *sint;
}

#endif
# 1036 "beam/erl_threads.h"

/* 32-bit atomics */

#define erts_atomic32_init_nob ethr_atomic32_init
#define erts_atomic32_set_nob ethr_atomic32_set
#define erts_atomic32_read_nob ethr_atomic32_read
#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
#define erts_atomic32_inc_nob ethr_atomic32_inc
#define erts_atomic32_dec_nob ethr_atomic32_dec
#define erts_atomic32_add_read_nob ethr_atomic32_add_read
#define erts_atomic32_add_nob ethr_atomic32_add
#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
#define erts_atomic32_read_band_nob ethr_atomic32_read_band
#define erts_atomic32_xchg_nob ethr_atomic32_xchg
#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg,
			    var, mask, set);
}
#endif
# 1065 "beam/erl_threads.h"

#define erts_atomic32_init_mb ethr_atomic32_init_mb
#define erts_atomic32_set_mb ethr_atomic32_set_mb
#define erts_atomic32_read_mb ethr_atomic32_read_mb
#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
#define erts_atomic32_add_mb ethr_atomic32_add_mb
#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 1092 "beam/erl_threads.h"

#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 1119 "beam/erl_threads.h"

#define erts_atomic32_init_relb ethr_atomic32_init_relb
#define erts_atomic32_set_relb ethr_atomic32_set_relb
#define erts_atomic32_read_relb ethr_atomic32_read_relb
#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
#define erts_atomic32_add_relb ethr_atomic32_add_relb
#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 1146 "beam/erl_threads.h"

#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 1173 "beam/erl_threads.h"

#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
#define erts_atomic32_add_rb ethr_atomic32_add_rb
#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 1200 "beam/erl_threads.h"

#define erts_atomic32_init_wb ethr_atomic32_init_wb
#define erts_atomic32_set_wb ethr_atomic32_set_wb
#define erts_atomic32_read_wb ethr_atomic32_read_wb
#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
#define erts_atomic32_add_wb ethr_atomic32_add_wb
#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1229 "beam/erl_threads.h"

#undef ERTS_ATOMIC_BSET_IMPL__

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    return *sint;
}

#endif
# 1249 "beam/erl_threads.h"

/* 64-bit atomics */

#if defined(ARCH_64)

#define erts_atomic64_init_nob ethr_atomic_init
#define erts_atomic64_set_nob ethr_atomic_set
#define erts_atomic64_read_nob ethr_atomic_read
#define erts_atomic64_inc_read_nob ethr_atomic_inc_read
#define erts_atomic64_dec_read_nob ethr_atomic_dec_read
#define erts_atomic64_inc_nob ethr_atomic_inc
#define erts_atomic64_dec_nob ethr_atomic_dec
#define erts_atomic64_add_read_nob ethr_atomic_add_read
#define erts_atomic64_add_nob ethr_atomic_add
#define erts_atomic64_read_bor_nob ethr_atomic_read_bor
#define erts_atomic64_read_band_nob ethr_atomic_read_band
#define erts_atomic64_xchg_nob ethr_atomic_xchg
#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob

#define erts_atomic64_init_mb ethr_atomic_init_mb
#define erts_atomic64_set_mb ethr_atomic_set_mb
#define erts_atomic64_read_mb ethr_atomic_read_mb
#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic64_inc_mb ethr_atomic_inc_mb
#define erts_atomic64_dec_mb ethr_atomic_dec_mb
#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic64_add_mb ethr_atomic_add_mb
#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb

#define erts_atomic64_init_acqb ethr_atomic_init_acqb
#define erts_atomic64_set_acqb ethr_atomic_set_acqb
#define erts_atomic64_read_acqb ethr_atomic_read_acqb
#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic64_add_acqb ethr_atomic_add_acqb
#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_atomic64_init_relb ethr_atomic_init_relb
#define erts_atomic64_set_relb ethr_atomic_set_relb
#define erts_atomic64_read_relb ethr_atomic_read_relb
#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic64_inc_relb ethr_atomic_inc_relb
#define erts_atomic64_dec_relb ethr_atomic_dec_relb
#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic64_add_relb ethr_atomic_add_relb
#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb

#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_atomic64_init_rb ethr_atomic_init_rb
#define erts_atomic64_set_rb ethr_atomic_set_rb
#define erts_atomic64_read_rb ethr_atomic_read_rb
#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic64_inc_rb ethr_atomic_inc_rb
#define erts_atomic64_dec_rb ethr_atomic_dec_rb
#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic64_add_rb ethr_atomic_add_rb
#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb

#define erts_atomic64_init_wb ethr_atomic_init_wb
#define erts_atomic64_set_wb ethr_atomic_set_wb
#define erts_atomic64_read_wb ethr_atomic_read_wb
#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic64_inc_wb ethr_atomic_inc_wb
#define erts_atomic64_dec_wb ethr_atomic_dec_wb
#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic64_add_wb ethr_atomic_add_wb
#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb

#define erts_atomic64_set_dirty erts_atomic_set_dirty
#define erts_atomic64_read_dirty erts_atomic_read_dirty

#elif defined(ARCH_32)
# 1363 "beam/erl_threads.h"

#undef ERTS_ATOMIC64_OPS_DECL__

#define ERTS_ATOMIC64_OPS_DECL__(BARRIER)				\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)

ERTS_ATOMIC64_OPS_DECL__(nob);
ERTS_ATOMIC64_OPS_DECL__(mb);
ERTS_ATOMIC64_OPS_DECL__(acqb);
ERTS_ATOMIC64_OPS_DECL__(relb);
ERTS_ATOMIC64_OPS_DECL__(ddrb);
ERTS_ATOMIC64_OPS_DECL__(rb);
ERTS_ATOMIC64_OPS_DECL__(wb);

#undef ERTS_ATOMIC64_OPS_DECL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

/*
 * The ethr_dw_atomic_*_nob() functions below
 * are here to make it possible for the
 * ERTS_ATOMIC64_OPS_IMPL__() to map erts
 * barriers to ethread barriers...
 */
static ERTS_INLINE void
ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_init(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
		       ethr_dw_sint_t *val)
{
    ethr_dw_atomic_set(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_read(var, val);
}

static ERTS_INLINE int
ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
			   ethr_dw_sint_t *new,
			   ethr_dw_sint_t *xchg)
{
    return ethr_dw_atomic_cmpxchg(var, new, xchg);
}

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#ifdef ETHR_SU_DW_NAINT_T__
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((erts_aint64_t) DW.dw_sint)
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
#else /* !ETHR_SU_DW_NAINT_T__ */
# 1470 "beam/erl_threads.h"
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32)		\
     | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD])		\
	& ((erts_aint64_t) 0xffffffff)))
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    do {								\
	DW.sint[ETHR_DW_SINT_LOW_WORD] =				\
	    (ethr_sint_t) (AINT64 & 0xffffffff);			\
	DW.sint[ETHR_DW_SINT_HIGH_WORD] = 				\
	    (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff);		\
    } while (0)
#endif /* !ETHR_SU_DW_NAINT_T__ */
# 1482 "beam/erl_threads.h"

#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, 			\
					AVarP, XchgVar, NewVar, 	\
					ModificationCode)		\
do {									\
    ethr_dw_sint_t dw_xchg__, dw_new__;					\
    ethr_dw_atomic_read(AVarP, &dw_xchg__);				\
    do {								\
	XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__);			\
	{								\
	    ModificationCode;						\
	}								\
	ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar);			\
    } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__));		\
} while (0)

#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER)				\
    									\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_init_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_set_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    ethr_dw_sint_t dw;							\
    ethr_dw_atomic_read_ ## BARRIER(var, &dw);				\
    return ERTS_DW_SINT_TO_AINT64__(dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg | val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg & val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = val);				\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp)			\
{									\
    ethr_dw_sint_t dw_xchg, dw_new;					\
    ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp);				\
    ERTS_AINT64_TO_DW_SINT__(dw_new, new);				\
    if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg))	\
	return exp;							\
    return ERTS_DW_SINT_TO_AINT64__(dw_xchg);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    {					\
					new = xchg & ~mask;		\
					new |= mask & set;		\
				    });					\
    return xchg;							\
}

ERTS_ATOMIC64_OPS_IMPL__(nob)
ERTS_ATOMIC64_OPS_IMPL__(mb)
ERTS_ATOMIC64_OPS_IMPL__(acqb)
ERTS_ATOMIC64_OPS_IMPL__(relb)
ERTS_ATOMIC64_OPS_IMPL__(ddrb)
ERTS_ATOMIC64_OPS_IMPL__(rb)
ERTS_ATOMIC64_OPS_IMPL__(wb)

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    ethr_dw_sint_t dw;
    ERTS_AINT64_TO_DW_SINT__(dw, val);
    sint[0] = dw.sint[0];
    sint[1] = dw.sint[1];
}

ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var)
{
    ethr_sint_t *sint;
    ethr_dw_sint_t dw;
    sint = ethr_dw_atomic_addr(var);
    dw.sint[0] = sint[0];
    dw.sint[1] = sint[1];
    return ERTS_DW_SINT_TO_AINT64__(dw);
}

#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1682 "beam/erl_threads.h"

#endif /* ARCH_32 */
# 1684 "beam/erl_threads.h"

#else /* !USE_THREADS */
# 1686 "beam/erl_threads.h"

/* Double word size atomics */

#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_atomic_init_nob erts_no_atomic_set
#define erts_atomic_set_nob erts_no_atomic_set
#define erts_atomic_read_nob erts_no_atomic_read
#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_atomic_inc_nob erts_no_atomic_inc
#define erts_atomic_dec_nob erts_no_atomic_dec
#define erts_atomic_add_read_nob erts_no_atomic_add_read
#define erts_atomic_add_nob erts_no_atomic_add
#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_atomic_read_band_nob erts_no_atomic_read_band
#define erts_atomic_xchg_nob erts_no_atomic_xchg
#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_atomic_init_mb erts_no_atomic_set
#define erts_atomic_set_mb erts_no_atomic_set
#define erts_atomic_read_mb erts_no_atomic_read
#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_atomic_inc_mb erts_no_atomic_inc
#define erts_atomic_dec_mb erts_no_atomic_dec
#define erts_atomic_add_read_mb erts_no_atomic_add_read
#define erts_atomic_add_mb erts_no_atomic_add
#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_atomic_read_band_mb erts_no_atomic_read_band
#define erts_atomic_xchg_mb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_atomic_init_acqb erts_no_atomic_set
#define erts_atomic_set_acqb erts_no_atomic_set
#define erts_atomic_read_acqb erts_no_atomic_read
#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_atomic_inc_acqb erts_no_atomic_inc
#define erts_atomic_dec_acqb erts_no_atomic_dec
#define erts_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_atomic_add_acqb erts_no_atomic_add
#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_atomic_init_relb erts_no_atomic_set
#define erts_atomic_set_relb erts_no_atomic_set
#define erts_atomic_read_relb erts_no_atomic_read
#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_atomic_inc_relb erts_no_atomic_inc
#define erts_atomic_dec_relb erts_no_atomic_dec
#define erts_atomic_add_read_relb erts_no_atomic_add_read
#define erts_atomic_add_relb erts_no_atomic_add
#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_atomic_read_band_relb erts_no_atomic_read_band
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_atomic_init_ddrb erts_no_atomic_set
#define erts_atomic_set_ddrb erts_no_atomic_set
#define erts_atomic_read_ddrb erts_no_atomic_read
#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_atomic_inc_ddrb erts_no_atomic_inc
#define erts_atomic_dec_ddrb erts_no_atomic_dec
#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_atomic_add_ddrb erts_no_atomic_add
#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
#define erts_atomic_read_rb erts_no_atomic_read
#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_atomic_inc_rb erts_no_atomic_inc
#define erts_atomic_dec_rb erts_no_atomic_dec
#define erts_atomic_add_read_rb erts_no_atomic_add_read
#define erts_atomic_add_rb erts_no_atomic_add
#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_atomic_read_band_rb erts_no_atomic_read_band
#define erts_atomic_xchg_rb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_atomic_init_wb erts_no_atomic_set
#define erts_atomic_set_wb erts_no_atomic_set
#define erts_atomic_read_wb erts_no_atomic_read
#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_atomic_inc_wb erts_no_atomic_inc
#define erts_atomic_dec_wb erts_no_atomic_dec
#define erts_atomic_add_read_wb erts_no_atomic_add_read
#define erts_atomic_add_wb erts_no_atomic_add
#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_atomic_read_band_wb erts_no_atomic_read_band
#define erts_atomic_xchg_wb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_atomic_set_dirty erts_no_atomic_set
#define erts_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_atomic32_init_nob erts_no_atomic32_set
#define erts_atomic32_set_nob erts_no_atomic32_set
#define erts_atomic32_read_nob erts_no_atomic32_read
#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_atomic32_inc_nob erts_no_atomic32_inc
#define erts_atomic32_dec_nob erts_no_atomic32_dec
#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_atomic32_add_nob erts_no_atomic32_add
#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_atomic32_init_mb erts_no_atomic32_set
#define erts_atomic32_set_mb erts_no_atomic32_set
#define erts_atomic32_read_mb erts_no_atomic32_read
#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_atomic32_inc_mb erts_no_atomic32_inc
#define erts_atomic32_dec_mb erts_no_atomic32_dec
#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_atomic32_add_mb erts_no_atomic32_add
#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_atomic32_init_acqb erts_no_atomic32_set
#define erts_atomic32_set_acqb erts_no_atomic32_set
#define erts_atomic32_read_acqb erts_no_atomic32_read
#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_atomic32_add_acqb erts_no_atomic32_add
#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_atomic32_init_relb erts_no_atomic32_set
#define erts_atomic32_set_relb erts_no_atomic32_set
#define erts_atomic32_read_relb erts_no_atomic32_read
#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_atomic32_inc_relb erts_no_atomic32_inc
#define erts_atomic32_dec_relb erts_no_atomic32_dec
#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_atomic32_add_relb erts_no_atomic32_add
#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_atomic32_init_ddrb erts_no_atomic32_set
#define erts_atomic32_set_ddrb erts_no_atomic32_set
#define erts_atomic32_read_ddrb erts_no_atomic32_read
#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_atomic32_add_ddrb erts_no_atomic32_add
#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
#define erts_atomic32_read_rb erts_no_atomic32_read
#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_atomic32_inc_rb erts_no_atomic32_inc
#define erts_atomic32_dec_rb erts_no_atomic32_dec
#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_atomic32_add_rb erts_no_atomic32_add
#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_atomic32_init_wb erts_no_atomic32_set
#define erts_atomic32_set_wb erts_no_atomic32_set
#define erts_atomic32_read_wb erts_no_atomic32_read
#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_atomic32_inc_wb erts_no_atomic32_inc
#define erts_atomic32_dec_wb erts_no_atomic32_dec
#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_atomic32_add_wb erts_no_atomic32_add
#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_atomic32_set_dirty erts_no_atomic32_set
#define erts_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_atomic64_init_nob erts_no_atomic64_set
#define erts_atomic64_set_nob erts_no_atomic64_set
#define erts_atomic64_read_nob erts_no_atomic64_read
#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_atomic64_inc_nob erts_no_atomic64_inc
#define erts_atomic64_dec_nob erts_no_atomic64_dec
#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_atomic64_add_nob erts_no_atomic64_add
#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_atomic64_init_mb erts_no_atomic64_set
#define erts_atomic64_set_mb erts_no_atomic64_set
#define erts_atomic64_read_mb erts_no_atomic64_read
#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_atomic64_inc_mb erts_no_atomic64_inc
#define erts_atomic64_dec_mb erts_no_atomic64_dec
#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_atomic64_add_mb erts_no_atomic64_add
#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_atomic64_init_acqb erts_no_atomic64_set
#define erts_atomic64_set_acqb erts_no_atomic64_set
#define erts_atomic64_read_acqb erts_no_atomic64_read
#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_atomic64_add_acqb erts_no_atomic64_add
#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_atomic64_init_relb erts_no_atomic64_set
#define erts_atomic64_set_relb erts_no_atomic64_set
#define erts_atomic64_read_relb erts_no_atomic64_read
#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_atomic64_inc_relb erts_no_atomic64_inc
#define erts_atomic64_dec_relb erts_no_atomic64_dec
#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_atomic64_add_relb erts_no_atomic64_add
#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_atomic64_init_ddrb erts_no_atomic64_set
#define erts_atomic64_set_ddrb erts_no_atomic64_set
#define erts_atomic64_read_ddrb erts_no_atomic64_read
#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_atomic64_add_ddrb erts_no_atomic64_add
#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_atomic64_init_rb erts_no_atomic64_set
#define erts_atomic64_set_rb erts_no_atomic64_set
#define erts_atomic64_read_rb erts_no_atomic64_read
#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_atomic64_inc_rb erts_no_atomic64_inc
#define erts_atomic64_dec_rb erts_no_atomic64_dec
#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_atomic64_add_rb erts_no_atomic64_add
#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_atomic64_init_wb erts_no_atomic64_set
#define erts_atomic64_set_wb erts_no_atomic64_set
#define erts_atomic64_read_wb erts_no_atomic64_read
#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_atomic64_inc_wb erts_no_atomic64_inc
#define erts_atomic64_dec_wb erts_no_atomic64_dec
#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_atomic64_add_wb erts_no_atomic64_add
#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_atomic64_set_dirty erts_no_atomic64_set
#define erts_atomic64_read_dirty erts_no_atomic64_read

#endif /* !USE_THREADS */
# 2058 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_thr_init(erts_thr_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_init(id);
    if (res)
	erts_thr_fatal_error(res, "initialize thread library");
#endif
# 2069 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_late_init(erts_thr_late_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_late_init(id);
    if (res)
	erts_thr_fatal_error(res, "complete initialization of thread library");
#endif
# 2079 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
		erts_thr_opts_t *opts)
{
#ifdef USE_THREADS
    int res = ethr_thr_create(tid, func, arg, opts);
    if (res)
	erts_thr_fatal_error(res, "create thread");
#endif
# 2090 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_join(erts_tid_t tid, void **thr_res)
{
#ifdef USE_THREADS
    int res = ethr_thr_join(tid, thr_res);
    if (res)
	erts_thr_fatal_error(res, "join thread");
#endif
# 2100 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_detach(erts_tid_t tid)
{
#ifdef USE_THREADS
    int res = ethr_thr_detach(tid);
    if (res)
	erts_thr_fatal_error(res, "detach thread");
#endif
# 2111 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_exit(void *res)
{
#ifdef USE_THREADS
    ethr_thr_exit(res);
    erts_thr_fatal_error(0, "terminate thread");
#endif
# 2121 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_install_exit_handler(void (*exit_handler)(void))
{
#ifdef USE_THREADS
    int res = ethr_install_exit_handler(exit_handler);
    if (res != 0)
	erts_thr_fatal_error(res, "install thread exit handler");
#endif
# 2131 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tid_t
erts_thr_self(void)
{
#ifdef USE_THREADS
    return ethr_self();
#else
# 2139 "beam/erl_threads.h"
    return 0;
#endif
# 2141 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
{
#ifdef USE_THREADS
    return ethr_getname(tid, buf, len);
#else
# 2149 "beam/erl_threads.h"
    return -1;
#endif
# 2151 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
erts_equal_tids(erts_tid_t x, erts_tid_t y)
{
#ifdef USE_THREADS
    return ethr_equal_tids(x, y);
#else
# 2160 "beam/erl_threads.h"
    return 1;
#endif
# 2162 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2174 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2180 "beam/erl_threads.h"
#endif
# 2181 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
		    int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2194 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
#endif
# 2200 "beam/erl_threads.h"
#endif
# 2201 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2214 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2220 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2224 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2227 "beam/erl_threads.h"
#endif
# 2228 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2240 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2243 "beam/erl_threads.h"
#endif
# 2244 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2256 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2259 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2263 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2266 "beam/erl_threads.h"
#endif
# 2267 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_destroy(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&mtx->lc);
#endif
# 2277 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&mtx->lcnt);
#endif
# 2280 "beam/erl_threads.h"
    res = ethr_mutex_destroy(&mtx->mtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy mutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2290 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy mutex");
    }
#endif
# 2293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2299 "beam/erl_threads.h"
erts_mtx_trylock(erts_mtx_t *mtx)
#endif
# 2301 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy(&mtx->lc))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2310 "beam/erl_threads.h"

    res = ethr_mutex_trylock(&mtx->mtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_x(res == 0, &mtx->lc,file,line);
#else
# 2317 "beam/erl_threads.h"
    erts_lc_trylock(res == 0, &mtx->lc);
#endif
# 2319 "beam/erl_threads.h"
#endif
# 2320 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, res);
#endif    
# 2323 "beam/erl_threads.h"
    return res;
#else
# 2325 "beam/erl_threads.h"
    return 0;
#endif
# 2327 "beam/erl_threads.h"

}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2334 "beam/erl_threads.h"
erts_mtx_lock(erts_mtx_t *mtx)
#endif
# 2336 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&mtx->lc, file, line);
#else
# 2342 "beam/erl_threads.h"
    erts_lc_lock(&mtx->lc);
#endif
# 2344 "beam/erl_threads.h"
#endif
# 2345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2348 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
#endif
# 2352 "beam/erl_threads.h"
#endif
# 2353 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_unlock(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2362 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2365 "beam/erl_threads.h"
    ethr_mutex_unlock(&mtx->mtx);
#endif
# 2367 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_mtx_is_locked(erts_mtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2379 "beam/erl_threads.h"
    return 0;
#endif
# 2381 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_init(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_init(cnd);
    if (res)
	erts_thr_fatal_error(res, "initialize condition variable");
#endif
# 2391 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_destroy(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_destroy(cnd);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy cond destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2407 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy condition variable");
    }
#endif
# 2410 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2420 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2423 "beam/erl_threads.h"
    res = ethr_cond_wait(cnd, &mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock(&mtx->lc);
#endif
# 2427 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2430 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post(&mtx->lcnt);
#endif
# 2433 "beam/erl_threads.h"
    if (res != 0 && res != EINTR)
	erts_thr_fatal_error(res, "wait on condition variable");
#endif
# 2436 "beam/erl_threads.h"
}

/*
 * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_cnd_signal(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_signal(cnd);
#endif
# 2454 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_cnd_broadcast(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_broadcast(cnd);
#endif
# 2463 "beam/erl_threads.h"
}

/* rwmutex */

ERTS_GLB_INLINE void
erts_rwmtx_set_reader_group(int no)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2475 "beam/erl_threads.h"
    res = ethr_rwmutex_set_reader_group(no);
    if (res != 0)
	erts_thr_fatal_error(res, "set reader group");
#endif
# 2479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
		      erts_rwmtx_opt_t *opt,
		      char *name,
		      Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
#endif
# 2494 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (name && name[0] == '\0')
      erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
    else
      erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
#endif
# 2500 "beam/erl_threads.h"
#endif
# 2501 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
		  char *name,
		  Eterm extra)
{
    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
		    erts_rwmtx_opt_t *opt,
		    char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2523 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
#endif
# 2526 "beam/erl_threads.h"
#endif
# 2527 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
{
    erts_rwmtx_init_opt(rwmtx, NULL, name);
}

ERTS_GLB_INLINE void
erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&rwmtx->lc);
#endif
# 2543 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&rwmtx->lcnt);
#endif
# 2546 "beam/erl_threads.h"
    res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwmutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2556 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwmutex");
    }
#endif
# 2559 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2565 "beam/erl_threads.h"
erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
#endif
# 2567 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2576 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2583 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2585 "beam/erl_threads.h"
#endif
# 2586 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
#endif
# 2589 "beam/erl_threads.h"
    
    return res;
#else
# 2592 "beam/erl_threads.h"
    return 0;
#endif
# 2594 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2600 "beam/erl_threads.h"
erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
#endif
# 2602 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2608 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2610 "beam/erl_threads.h"
#endif
# 2611 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2614 "beam/erl_threads.h"
    ethr_rwmutex_rlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2618 "beam/erl_threads.h"
#endif
# 2619 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2628 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2631 "beam/erl_threads.h"
    ethr_rwmutex_runlock(&rwmtx->rwmtx);
#endif
# 2633 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2640 "beam/erl_threads.h"
erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
#endif
# 2642 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2651 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2658 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2660 "beam/erl_threads.h"
#endif
# 2661 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2664 "beam/erl_threads.h"
    
    return res;
#else
# 2667 "beam/erl_threads.h"
    return 0;
#endif
# 2669 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2675 "beam/erl_threads.h"
erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
#endif
# 2677 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2683 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2685 "beam/erl_threads.h"
#endif
# 2686 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2689 "beam/erl_threads.h"
    ethr_rwmutex_rwlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2693 "beam/erl_threads.h"
#endif
# 2694 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2703 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2706 "beam/erl_threads.h"
    ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
#endif
# 2708 "beam/erl_threads.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_rwmtx_trywlock(erts_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wlock(erts_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx)
{

}

#endif
# 2735 "beam/erl_threads.h"

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2746 "beam/erl_threads.h"
    return 0;
#endif
# 2748 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2760 "beam/erl_threads.h"
    return 0;
#endif
# 2762 "beam/erl_threads.h"
}

/* No atomic ops */

ERTS_GLB_INLINE void
erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    var->sint[0] = val->sint[0];
    var->sint[1] = val->sint[1];
}

ERTS_GLB_INLINE void
erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    val->sint[0] = var->sint[0];
    val->sint[1] = var->sint[1];
}

ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *new_val,
					      erts_no_dw_atomic_t *old_val)
{
    if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
	erts_no_dw_atomic_read(var, old_val);
	return 0;
    }
    else {
	erts_no_dw_atomic_set(var, new_val);
	return !0;
    }
}

ERTS_GLB_INLINE void
erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read(erts_no_atomic_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_inc_read(erts_no_atomic_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_dec_read(erts_no_atomic_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic_inc(erts_no_atomic_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic_dec(erts_no_atomic_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
{
    erts_aint_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
		       erts_aint_t new,
		       erts_aint_t expected)
{
    erts_aint_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bset(erts_no_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    erts_aint_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic32 */

ERTS_GLB_INLINE void
erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read(erts_no_atomic32_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_inc(erts_no_atomic32_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_dec(erts_no_atomic32_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
{
    erts_aint32_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
			 erts_aint32_t new,
			 erts_aint32_t expected)
{
    erts_aint32_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    erts_aint32_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic64 */

ERTS_GLB_INLINE void
erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read(erts_no_atomic64_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_inc(erts_no_atomic64_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_dec(erts_no_atomic64_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
{
    erts_aint64_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
			 erts_aint64_t new,
			 erts_aint64_t expected)
{
    erts_aint64_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
			   erts_aint64_t mask,
			   erts_aint64_t set)
{
    erts_aint64_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* spinlock */

ERTS_GLB_INLINE void
erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3098 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
#endif
# 3101 "beam/erl_threads.h"
#else
# 3102 "beam/erl_threads.h"
    (void)lock;
#endif
# 3104 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
			 Uint16 opt)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3117 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
#endif
# 3120 "beam/erl_threads.h"
#else
# 3121 "beam/erl_threads.h"
    (void)lock;
#endif
# 3123 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_spinlock_init(erts_spinlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK);
#endif
# 3136 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
#endif
# 3139 "beam/erl_threads.h"
#else
# 3140 "beam/erl_threads.h"
    (void)lock;
#endif
# 3142 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_destroy(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3152 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3155 "beam/erl_threads.h"
    res = ethr_spinlock_destroy(&lock->slck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy spinlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3165 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3168 "beam/erl_threads.h"
    (void)lock;
#endif
# 3170 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spin_unlock(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&lock->lc);
#endif
# 3179 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&lock->lcnt);
#endif
# 3182 "beam/erl_threads.h"
    ethr_spin_unlock(&lock->slck);
#else
# 3184 "beam/erl_threads.h"
    (void)lock;
#endif
# 3186 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line)
#else
# 3192 "beam/erl_threads.h"
erts_spin_lock(erts_spinlock_t *lock)
#endif
# 3194 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&lock->lc,file,line);
#else
# 3200 "beam/erl_threads.h"
    erts_lc_lock(&lock->lc);
#endif
# 3202 "beam/erl_threads.h"
#endif
# 3203 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&lock->lcnt);
#endif
# 3206 "beam/erl_threads.h"
    ethr_spin_lock(&lock->slck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3210 "beam/erl_threads.h"
#else
# 3211 "beam/erl_threads.h"
    (void)lock;
#endif
# 3213 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3225 "beam/erl_threads.h"
    return 0;
#endif
# 3227 "beam/erl_threads.h"
}

/* rwspinlock */

ERTS_GLB_INLINE void
erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
#endif
# 3241 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
#endif
# 3244 "beam/erl_threads.h"
#else
# 3245 "beam/erl_threads.h"
    (void)lock;
#endif
# 3247 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_init(erts_rwlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK);
#endif
# 3259 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
#endif
# 3262 "beam/erl_threads.h"
#else
# 3263 "beam/erl_threads.h"
    (void)lock;
#endif
# 3265 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_destroy(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3275 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3278 "beam/erl_threads.h"
    res = ethr_rwlock_destroy(&lock->rwlck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3288 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3291 "beam/erl_threads.h"
    (void)lock;
#endif
# 3293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_read_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3302 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3305 "beam/erl_threads.h"
    ethr_read_unlock(&lock->rwlck);
#else
# 3307 "beam/erl_threads.h"
    (void)lock;
#endif
# 3309 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3315 "beam/erl_threads.h"
erts_read_lock(erts_rwlock_t *lock)
#endif
# 3317 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 3323 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3325 "beam/erl_threads.h"
#endif
# 3326 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3329 "beam/erl_threads.h"
    ethr_read_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3333 "beam/erl_threads.h"
#else
# 3334 "beam/erl_threads.h"
    (void)lock;
#endif
# 3336 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_write_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3348 "beam/erl_threads.h"
    ethr_write_unlock(&lock->rwlck);
#else
# 3350 "beam/erl_threads.h"
    (void)lock;
#endif
# 3352 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3358 "beam/erl_threads.h"
erts_write_lock(erts_rwlock_t *lock)
#endif
# 3360 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 3366 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3368 "beam/erl_threads.h"
#endif
# 3369 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3372 "beam/erl_threads.h"
    ethr_write_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3376 "beam/erl_threads.h"
#else
# 3377 "beam/erl_threads.h"
    (void)lock;
#endif
# 3379 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3391 "beam/erl_threads.h"
    return 0;
#endif
# 3393 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3405 "beam/erl_threads.h"
    return 0;
#endif
# 3407 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_create(keyp, keyname);
    if (res)
	erts_thr_fatal_error(res, "create thread specific data key");
#endif
# 3417 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_delete(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_delete(key);
    if (res)
	erts_thr_fatal_error(res, "delete thread specific data key");
#endif
# 3427 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_set(erts_tsd_key_t key, void *value)
{
#ifdef USE_THREADS
    int res = ethr_tsd_set(key, value);
    if (res)
	erts_thr_fatal_error(res, "set thread specific data");
#endif
# 3437 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void *
erts_tsd_get(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    return ethr_tsd_get(key);
#else
# 3445 "beam/erl_threads.h"
    return NULL;
#endif
# 3447 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
{
#ifdef USE_THREADS
    return (erts_tse_t *) ethr_get_ts_event();
#else
# 3454 "beam/erl_threads.h"
    return (erts_tse_t *) NULL;
#endif
# 3456 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_leave_ts_event(ep);
#endif
# 3463 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
{
#ifdef USE_THREADS
    int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
    if (res != 0)
	erts_thr_fatal_error(res, "prepare timed");
#endif
# 3472 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_set(&((ethr_ts_event *) ep)->event);
#endif
# 3479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_reset(&((ethr_ts_event *) ep)->event);
#endif
# 3486 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
#else
# 3493 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3495 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
{
#ifdef USE_THREADS
    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
#else
# 3502 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3504 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_twait(&((ethr_ts_event *) ep)->event,
			    (ethr_sint64_t) tmo);
#else
# 3512 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3514 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_stwait(&((ethr_ts_event *) ep)->event,
			     spincount,
			     (ethr_sint64_t) tmo);
#else
# 3523 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3525 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
#else
# 3532 "beam/erl_threads.h"
    return 0;
#endif
# 3534 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
{
#ifdef USE_THREADS
    int res = ethr_set_main_thr_status(on, no);
    if (res != 0)
	erts_thr_fatal_error(res, "set thread main status");
#endif
# 3543 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_thr_get_main_status(void)
{
#ifdef USE_THREADS
    int main_status;
    int res = ethr_get_main_thr_status(&main_status);
    if (res != 0)
	erts_thr_fatal_error(res, "get thread main status");
    return main_status;
#else
# 3554 "beam/erl_threads.h"
    return 1;
#endif
# 3556 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_yield(void)
{
#ifdef USE_THREADS
    int res = ETHR_YIELD();
    if (res != 0)
	erts_thr_fatal_error(res, "yield");
#endif    
# 3565 "beam/erl_threads.h"
}


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS

ERTS_GLB_INLINE void
erts_thr_kill(erts_tid_t tid, int sig) {
#ifdef USE_THREADS
  int res = ethr_kill((ethr_tid)tid, sig);
  if (res)
    erts_thr_fatal_error(res, "killing thread");
#endif
# 3577 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef USE_THREADS
    int res = ethr_sigmask(how, set, oset);
    if (res)
	erts_thr_fatal_error(res, "get or set signal mask");
#endif
# 3587 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef USE_THREADS
    int res;
    do {
	res = ethr_sigwait(set, sig);
    } while (res == EINTR);
    if (res)
	erts_thr_fatal_error(res, "to wait for signal");
#endif
# 3600 "beam/erl_threads.h"
}

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 3603 "beam/erl_threads.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 3605 "beam/erl_threads.h"

#endif /* #ifndef ERL_THREAD_H__ */
# 3607 "beam/erl_threads.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_atomic_init
#undef erts_atomic_set
#undef erts_atomic_read
#undef erts_atomic_inctest
#undef erts_atomic_dectest
#undef erts_atomic_inc
#undef erts_atomic_dec
#undef erts_atomic_addtest
#undef erts_atomic_add
#undef erts_atomic_xchg
#undef erts_atomic_cmpxchg
#undef erts_atomic_bor
#undef erts_atomic_band

#undef erts_atomic32_init
#undef erts_atomic32_set
#undef erts_atomic32_read
#undef erts_atomic32_inctest
#undef erts_atomic32_dectest
#undef erts_atomic32_inc
#undef erts_atomic32_dec
#undef erts_atomic32_addtest
#undef erts_atomic32_add
#undef erts_atomic32_xchg
#undef erts_atomic32_cmpxchg
#undef erts_atomic32_bor
#undef erts_atomic32_band

#endif
# 3641 "beam/erl_threads.h"
# 29 "beam/erl_smp.h" 2

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 492 "beam/sys.h" 2

#ifdef ERTS_WANT_BREAK_HANDLING
#  ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_break_requested;
#    define ERTS_BREAK_REQUESTED \
  ((int) erts_smp_atomic32_read_nob(&erts_break_requested))
#  else
# 499 "beam/sys.h"
extern volatile int erts_break_requested;
#    define ERTS_BREAK_REQUESTED erts_break_requested
#  endif
# 502 "beam/sys.h"
void erts_do_break_handling(void);
#endif
# 504 "beam/sys.h"

#ifdef ERTS_WANT_GOT_SIGUSR1
#  ifndef UNIX
#    define ERTS_GOT_SIGUSR1 0
#  else
# 509 "beam/sys.h"
#    ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read_mb(&erts_got_sigusr1))
#    else
# 513 "beam/sys.h"
extern volatile int erts_got_sigusr1;
#      define ERTS_GOT_SIGUSR1 erts_got_sigusr1
#    endif
# 516 "beam/sys.h"
#  endif
# 517 "beam/sys.h"
#endif
# 518 "beam/sys.h"

#ifdef ERTS_SMP
extern erts_smp_atomic32_t erts_writing_erl_crash_dump;
extern erts_tsd_key_t erts_is_crash_dumping_key;
#define ERTS_SOMEONE_IS_CRASH_DUMPING \
  ((int) erts_smp_atomic32_read_mb(&erts_writing_erl_crash_dump))
#define ERTS_IS_CRASH_DUMPING \
  ((int) (SWord) erts_tsd_get(erts_is_crash_dumping_key))
#else
# 527 "beam/sys.h"
extern volatile int erts_writing_erl_crash_dump;
#define ERTS_SOMEONE_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#define ERTS_IS_CRASH_DUMPING erts_writing_erl_crash_dump
#endif
# 531 "beam/sys.h"

/* Deal with memcpy() vs bcopy() etc. We want to use the mem*() functions,
   but be able to fall back on bcopy() etc on systems that don't have
   mem*(), but this doesn't work to well with memset()/bzero() - thus the
   memzero() macro.
*/

/* xxxP */
#if defined(USE_BCOPY)
#  define memcpy(a, b, c) bcopy((b), (a), (c))
#  define memcmp(a, b, c) bcmp((a), (b), (c))
#  define memzero(buf, len) bzero((buf), (len))
#else
# 544 "beam/sys.h"
#  define memzero(buf, len) memset((buf), '\0', (len))
#endif
# 546 "beam/sys.h"

/* Stuff that is useful for port programs, drivers, etc */

#ifdef ISC32			/* Too much for the Makefile... */
#  define signal	sigset
#  define NO_ASINH
#  define NO_ACOSH
#  define NO_ATANH
#  define NO_FTRUNCATE
#  define SIG_SIGHOLD
#  define _POSIX_SOURCE 
#  define _XOPEN_SOURCE
#endif
# 559 "beam/sys.h"

#ifdef QNX			/* Too much for the Makefile... */
#  define SYS_SELECT_H
#  define NO_ERF
#  define NO_ERFC
/* This definition doesn't take NaN into account, but matherr() gets those */
#  define isfinite(x) (fabs(x) != HUGE_VAL)
#  define USE_MATHERR
#  define HAVE_FINITE
#endif
# 569 "beam/sys.h"


#ifdef WANT_NONBLOCKING	    /* must define this to pull in fcntl.h/ioctl.h */

/* This is really a mess... We used to use fcntl O_NDELAY, but that seems
   to only work on SunOS 4 - in particular, on SysV-based systems
   (including Solaris 2), it does set non-blocking mode, but causes
   read() to return 0!!  fcntl O_NONBLOCK is specified by POSIX, and
   seems to work on most systems, with the notable exception of AIX,
   where the old ioctl FIONBIO is the *only* one that will set a *socket*
   in non-blocking mode - and ioctl FIONBIO on AIX *doesn't* work for
   pipes or ttys (O_NONBLOCK does)!!! For now, we'll use FIONBIO for AIX. */

#  ifdef __WIN32__

static unsigned long zero_value = 0, one_value = 1;
#    define SET_BLOCKING(fd)	{ if (ioctlsocket((fd), FIONBIO, &zero_value) != 0) fprintf(stderr, "Error setting socket to non-blocking: %d\n", WSAGetLastError()); }
#    define SET_NONBLOCKING(fd)	ioctlsocket((fd), FIONBIO, &one_value)

#  else
# 589 "beam/sys.h"
#    ifdef NB_FIONBIO		/* Old BSD */
#if 0 /* expanded by -frewrite-includes */
#      include <sys/ioctl.h>
#endif /* expanded by -frewrite-includes */
# 590 "beam/sys.h"
# 591 "beam/sys.h"
  static const int zero_value = 0, one_value = 1;
#      define SET_BLOCKING(fd)         ioctl((fd), FIONBIO, &zero_value)
#      define SET_NONBLOCKING(fd)      ioctl((fd), FIONBIO, &one_value)
#      define ERRNO_BLOCK EWOULDBLOCK
#    else /* !NB_FIONBIO */
# 596 "beam/sys.h"
#if 0 /* expanded by -frewrite-includes */
#      include <fcntl.h>
#endif /* expanded by -frewrite-includes */
# 596 "beam/sys.h"
# 597 "beam/sys.h"
#      ifdef NB_O_NDELAY               /* Nothing needs this? */
#        define NB_FLAG O_NDELAY
#        ifndef ERRNO_BLOCK            /* allow override (e.g. EAGAIN) via Makefile */
#          define ERRNO_BLOCK EWOULDBLOCK
#        endif
# 602 "beam/sys.h"
#      else  /* !NB_O_NDELAY */	/* The True Way - POSIX!:-) */
# 603 "beam/sys.h"
#        define NB_FLAG O_NONBLOCK
#        define ERRNO_BLOCK EAGAIN
#      endif /* !NB_O_NDELAY */
# 606 "beam/sys.h"
#      define SET_BLOCKING(fd)         fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) & ~NB_FLAG)
#      define SET_NONBLOCKING(fd)      fcntl((fd), F_SETFL, \
                                           fcntl((fd), F_GETFL, 0) | NB_FLAG)
#    endif /* !NB_FIONBIO */
# 611 "beam/sys.h"
#  endif /* !__WIN32__ */
# 612 "beam/sys.h"
#endif /* WANT_NONBLOCKING */
# 613 "beam/sys.h"

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);

/* Some special erts_exit() codes: */
#define ERTS_INTR_EXIT	-1		/* called from signal handler */
#define ERTS_ABORT_EXIT	-2	        /* no crash dump; only abort() */
#define ERTS_DUMP_EXIT	-3              /* crash dump; then exit() */
#define ERTS_ERROR_EXIT	-4              /* crash dump; then abort() */

#define ERTS_INTERNAL_ERROR(What) \
    erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \
	     __FILE__, __LINE__, __func__, What)

Eterm erts_check_io_info(void *p);

/* Size of misc memory allocated from system dependent code */
Uint erts_sys_misc_mem_sz(void);

/* print stuff is declared here instead of in global.h, so sys stuff won't
   have to include global.h */
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 633 "beam/sys.h"
# 1 "../include/internal/erl_printf.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2005-2009. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef ERL_PRINTF_H_
#define ERL_PRINTF_H_
#if 0 /* expanded by -frewrite-includes */
#include <stdio.h>
#endif /* expanded by -frewrite-includes */
# 23 "../include/internal/erl_printf.h"
# 24 "../include/internal/erl_printf.h"
#if 0 /* expanded by -frewrite-includes */
#include <stdarg.h>
#endif /* expanded by -frewrite-includes */
# 24 "../include/internal/erl_printf.h"
# 25 "../include/internal/erl_printf.h"

extern int (*erts_printf_stdout_func)(char *, va_list);
extern int (*erts_printf_stderr_func)(char *, va_list);
extern int erts_printf_add_cr_to_stdout;
extern int erts_printf_add_cr_to_stderr;
extern int (*erts_printf_block_fpe)(void);
extern void (*erts_printf_unblock_fpe)(int);

typedef struct erts_dsprintf_buf_t_ erts_dsprintf_buf_t;

struct erts_dsprintf_buf_t_ {
    char *str;
    size_t str_len;
    size_t size;
    erts_dsprintf_buf_t *(*grow)(erts_dsprintf_buf_t *, size_t);
};

#define ERTS_DSPRINTF_BUF_INITER(GFUNC) {NULL, 0, 0, (GFUNC)}

int erts_printf(const char *, ...);
int erts_fprintf(FILE *, const char *, ...);
int erts_fdprintf(int, const char *, ...);
int erts_sprintf(char *, const char *, ...);
int erts_snprintf(char *, size_t, const char *, ...);
int erts_dsprintf(erts_dsprintf_buf_t *, const char *, ...);

int erts_vprintf(const char *, va_list);
int erts_vfprintf(FILE *, const char *, va_list);
int erts_vfdprintf(int, const char *, va_list);
int erts_vsprintf(char *, const char *, va_list);
int erts_vsnprintf(char *, size_t, const char *, va_list);
int erts_vdsprintf(erts_dsprintf_buf_t *, const char *, va_list);

#endif /* #ifndef ERL_PRINTF_H_ */
# 59 "../include/internal/erl_printf.h"
# 634 "beam/sys.h" 2

/* Io constants to erts_print and erts_putc */
#define ERTS_PRINT_STDERR	(2)
#define ERTS_PRINT_STDOUT	(1)
#define ERTS_PRINT_FILE		(-1)
#define ERTS_PRINT_SBUF		(-2)
#define ERTS_PRINT_SNBUF	(-3)
#define ERTS_PRINT_DSBUF	(-4)

#define ERTS_PRINT_MIN		ERTS_PRINT_DSBUF

typedef struct {
    char *buf;
    size_t size;
} erts_print_sn_buf;

int erts_print(int to, void *arg, char *format, ...);	/* in utils.c */
int erts_putc(int to, void *arg, char);			/* in utils.c */

/* logger stuff is declared here instead of in global.h, so sys files
   won't have to include global.h */

erts_dsprintf_buf_t *erts_create_logger_dsbuf(void);
int erts_send_info_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_warning_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_to_logger(Eterm, erts_dsprintf_buf_t *);
int erts_send_error_term_to_logger(Eterm, erts_dsprintf_buf_t *, Eterm);
int erts_send_info_to_logger_str(Eterm, char *); 
int erts_send_warning_to_logger_str(Eterm, char *);
int erts_send_error_to_logger_str(Eterm, char *);
int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *);
int erts_send_info_to_logger_str_nogl(char *);
/* needed by erl_smp.h (declared above)
   int erts_send_warning_to_logger_str_nogl(char *); */
int erts_send_error_to_logger_str_nogl(char *);

typedef struct preload {
    char *name;			/* Name of module */
    int  size;			/* Size of code */
    unsigned char* code;	/* Code pointer */
} Preload;


/*
 * This structure contains options to all built in drivers.
 * None of the drivers use all of the fields.
 */

typedef struct _SysDriverOpts {
    Uint ifd;			/* Input file descriptor (fd driver). */
    Uint ofd;			/* Outputfile descriptor (fd driver). */
    int packet_bytes;		/* Number of bytes in packet header. */
    int read_write;		/* Read and write bits. */
    int use_stdio;		/* Use standard I/O: TRUE or FALSE. */
    int redir_stderr;           /* Redirect stderr to stdout: TRUE/FALSE. */
    int hide_window;		/* Hide this windows (Windows). */
    int exit_status;		/* Report exit status of subprocess. */
    int overlapped_io;          /* Only has effect on windows NT et al */
    char *envir;		/* Environment of the port process, */
				/* in Windows format. */
    char **argv;                /* Argument vector in Unix'ish format. */
    char *wd;			/* Working directory. */
    unsigned spawn_type;        /* Bitfield of ERTS_SPAWN_DRIVER | 
				   ERTS_SPAWN_EXTERNAL | both*/ 
    int parallelism;            /* Optimize for parallelism */
} SysDriverOpts;

extern char *erts_default_arg0;

extern char os_type[];

typedef struct {
    int have_os_monotonic_time;
    int have_corrected_os_monotonic_time;
    ErtsMonotonicTime os_monotonic_time_unit;
    ErtsMonotonicTime sys_clock_resolution;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
	int extended;
    } os_monotonic_time_info;
    struct {
	Uint64 resolution;
	char *func;
	char *clock_id;
	int locked_use;
    } os_system_time_info;
} ErtsSysInitTimeResult;

#define ERTS_SYS_INIT_TIME_RESULT_INITER \
    {0, 0, (ErtsMonotonicTime) -1, (ErtsMonotonicTime) 1}

extern void erts_init_sys_time_sup(void);
extern void sys_init_time(ErtsSysInitTimeResult *);
extern void erts_late_sys_init_time(void);
extern void erts_deliver_time(void);
extern void erts_time_remaining(SysTimeval *);
extern void erts_sys_init_float(void);
extern void erts_thread_init_float(void);
extern void erts_thread_disable_fpe(void);
ERTS_GLB_INLINE int erts_block_fpe(void);
ERTS_GLB_INLINE void erts_unblock_fpe(int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int erts_block_fpe(void)
{
    return erts_sys_block_fpe();
}

ERTS_GLB_INLINE void erts_unblock_fpe(int unmasked)
{
    erts_sys_unblock_fpe(unmasked);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 754 "beam/sys.h"


/* Dynamic library/driver loading */
typedef struct {
    char* str;
}ErtsSysDdllError;
#define ERTS_SYS_DDLL_ERROR_INIT {NULL}
extern void erts_sys_ddll_free_error(ErtsSysDdllError*);
extern void erl_sys_ddll_init(void); /* to initialize mutexes etc */
extern int erts_sys_ddll_open(const char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_open_noext(char *path, void **handle, ErtsSysDdllError*);
extern int erts_sys_ddll_load_driver_init(void *handle, void **function);
extern int erts_sys_ddll_load_nif_init(void *handle, void **function,ErtsSysDdllError*);
extern int erts_sys_ddll_close2(void *handle, ErtsSysDdllError*);
#define erts_sys_ddll_close(H) erts_sys_ddll_close2(H,NULL)
extern void *erts_sys_ddll_call_init(void *function);
extern void *erts_sys_ddll_call_nif_init(void *function);
extern int erts_sys_ddll_sym2(void *handle, const char *name, void **function, ErtsSysDdllError*);
#define erts_sys_ddll_sym(H,N,F) erts_sys_ddll_sym2(H,N,F,NULL)
extern char *erts_sys_ddll_error(int code);



/*
 * System interfaces for startup.
 */
void erts_sys_schedule_interrupt(int set);
#ifdef ERTS_SMP
void erts_sys_schedule_interrupt_timed(int, ErtsMonotonicTime);
void erts_sys_main_thread(void);
#endif
# 785 "beam/sys.h"

extern int erts_sys_prepare_crash_dump(int secs);
extern void erts_sys_pre_init(void);
extern void erl_sys_init(void);
extern void erl_sys_args(int *argc, char **argv);
extern void erl_sys_schedule(int);
void sys_tty_reset(int);

int sys_max_files(void);
void sys_init_io(void);
Preload* sys_preloaded(void);
unsigned char* sys_preload_begin(Preload*);
void sys_preload_end(Preload*);
int sys_get_key(int);
void elapsed_time_both(UWord *ms_user, UWord *ms_sys, 
		       UWord *ms_user_diff, UWord *ms_sys_diff);
void wall_clock_elapsed_time_both(UWord *ms_total, 
				  UWord *ms_diff);
void get_time(int *hour, int *minute, int *second);
void get_date(int *year, int *month, int *day);
void get_localtime(int *year, int *month, int *day, 
		   int *hour, int *minute, int *second);
void get_universaltime(int *year, int *month, int *day, 
		       int *hour, int *minute, int *second);
int seconds_to_univ(Sint64 seconds, 
		    Sint *year, Sint *month, Sint *day, 
		    Sint *hour, Sint *minute, Sint *second);
int univ_to_seconds(Sint year, Sint month, Sint day, 
		    Sint hour, Sint minute, Sint second,
		    Sint64* seconds);
int univ_to_local(
    Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second);
int local_to_univ(Sint *year, Sint *month, Sint *day, 
		  Sint *hour, Sint *minute, Sint *second, int isdst);
void get_now(Uint*, Uint*, Uint*);
struct ErtsSchedulerData_;
ErtsMonotonicTime erts_get_monotonic_time(struct ErtsSchedulerData_ *);
ErtsMonotonicTime erts_get_time_offset(void);
void
erts_make_timestamp_value(Uint* megasec, Uint* sec, Uint* microsec,
			  ErtsMonotonicTime mtime, ErtsMonotonicTime offset);
void get_sys_now(Uint*, Uint*, Uint*);
void set_break_quit(void (*)(void), void (*)(void));

void os_flavor(char*, unsigned);
void os_version(int*, int*, int*);
void init_getenv_state(GETENV_STATE *);
char * getenv_string(GETENV_STATE *);
void fini_getenv_state(GETENV_STATE *);

#define HAVE_ERTS_CHECK_IO_DEBUG
typedef struct {
    int no_used_fds;
    int no_driver_select_structs;
    int no_driver_event_structs;
} ErtsCheckIoDebugInfo;
int erts_check_io_debug(ErtsCheckIoDebugInfo *ip);

int erts_sys_is_area_readable(char *start, char *stop);

/* xxxP */
#define SYS_DEFAULT_FLOAT_DECIMALS 20
void init_sys_float(void);
int sys_chars_to_double(char*, double*);
int sys_double_to_chars(double, char*, size_t);
int sys_double_to_chars_ext(double, char*, size_t, size_t);
int sys_double_to_chars_fast(double, char*, int, int, int);
void sys_get_pid(char *, size_t);

/* erts_sys_putenv() returns, 0 on success and a value != 0 on failure. */
int erts_sys_putenv(char *key, char *value);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_putenv_raw(char *key, char *value);
/* erts_sys_getenv() returns 0 on success (length of value string in
   *size), a value > 0 if value buffer is too small (*size is set to needed
   size), and a value < 0 on failure. */
int erts_sys_getenv(char *key, char *value, size_t *size);
/* Simple variant used from drivers, raw eightbit interface */
int erts_sys_getenv_raw(char *key, char *value, size_t *size);
/* erts_sys_getenv__() is only allowed to be used in early init phase */
int erts_sys_getenv__(char *key, char *value, size_t *size);
/* erst_sys_unsetenv() returns 0 on success and a value != 0 on failure. */
int erts_sys_unsetenv(char *key);

/* Easier to use, but not as efficient, environment functions */
char *erts_read_env(char *key);
void erts_free_read_env(void *value);

#if defined(ERTS_THR_HAVE_SIG_FUNCS) && !defined(ETHR_UNUSABLE_SIGUSRX)
extern void sys_thr_resume(erts_tid_t tid);
extern void sys_thr_suspend(erts_tid_t tid);
#endif
# 878 "beam/sys.h"

/* utils.c */

/* Options to sys_alloc_opt */
#define SYS_ALLOC_OPT_TRIM_THRESHOLD 0
#define SYS_ALLOC_OPT_TOP_PAD        1
#define SYS_ALLOC_OPT_MMAP_THRESHOLD 2
#define SYS_ALLOC_OPT_MMAP_MAX       3

/* Default values to sys_alloc_opt options */
#define ERTS_DEFAULT_TRIM_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_TOP_PAD         0
#define ERTS_DEFAULT_MMAP_THRESHOLD  (128 * 1024)
#define ERTS_DEFAULT_MMAP_MAX        64

int sys_alloc_opt(int, int);

typedef struct {
  int trim_threshold;
  int top_pad;
  int mmap_threshold;
  int mmap_max;
} SysAllocStat;

void sys_alloc_stat(SysAllocStat *);

#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
#undef ERTS_REFC_DEBUG
#define ERTS_REFC_DEBUG
#endif
# 908 "beam/sys.h"

typedef erts_smp_atomic_t erts_refc_t;

ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val);
ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp,
					      erts_aint_t min_val);
ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff,
				   erts_aint_t min_val);
ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp,
					   erts_aint_t min_val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_refc_init(erts_refc_t *refcp, erts_aint_t val)
{
    erts_smp_atomic_init_nob((erts_smp_atomic_t *) refcp, val);
}

ERTS_GLB_INLINE void
erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 941 "beam/sys.h"
    erts_smp_atomic_inc_nob((erts_smp_atomic_t *) refcp);
#endif
# 943 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 955 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#else
# 968 "beam/sys.h"
    erts_smp_atomic_dec_nob((erts_smp_atomic_t *) refcp);
#endif
# 970 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 982 "beam/sys.h"
    return val;
}

ERTS_GLB_INLINE void
erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val)
{
#ifdef ERTS_REFC_DEBUG
    erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff);
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n",
		 diff, val, min_val);
#else
# 995 "beam/sys.h"
    erts_smp_atomic_add_nob((erts_smp_atomic_t *) refcp, diff);
#endif
# 997 "beam/sys.h"
}

ERTS_GLB_INLINE erts_aint_t
erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val)
{
    erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp);
#ifdef ERTS_REFC_DEBUG
    if (val < min_val)
	erts_exit(ERTS_ABORT_EXIT,
		 "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n",
		 val, min_val);
#endif
# 1009 "beam/sys.h"
    return val;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1013 "beam/sys.h"

#ifdef ERTS_ENABLE_KERNEL_POLL
extern int erts_use_kernel_poll;
#endif
# 1017 "beam/sys.h"

#define sys_memcpy(s1,s2,n)  memcpy(s1,s2,n)
#define sys_memmove(s1,s2,n) memmove(s1,s2,n)
#define sys_memcmp(s1,s2,n)  memcmp(s1,s2,n)
#define sys_memset(s,c,n)    memset(s,c,n)
#define sys_memzero(s, n)    memset(s,'\0',n)
#define sys_strcmp(s1,s2)    strcmp(s1,s2)
#define sys_strncmp(s1,s2,n) strncmp(s1,s2,n)
#define sys_strcpy(s1,s2)    strcpy(s1,s2)
#define sys_strncpy(s1,s2,n) strncpy(s1,s2,n)
#define sys_strlen(s)        strlen(s)

/* define function symbols (needed in sys_drv_api) */
#define sys_fp_alloc     sys_alloc
#define sys_fp_realloc   sys_realloc
#define sys_fp_free      sys_free
#define sys_fp_memcpy    memcpy
#define sys_fp_memmove   memmove
#define sys_fp_memcmp    memcmp
#define sys_fp_memset    memset
/* #define sys_fp_memzero    elib_memzero */
#define sys_fp_strcmp    strcmp
#define sys_fp_strncmp   strncmp
#define sys_fp_strcpy    strcpy
#define sys_fp_strncpy   strncpy
#define sys_fp_strlen    strlen


/* Return codes from the nb_read and nb_write functions */
#define FD_READY 1
#define FD_CONTINUE 2
#define FD_ERROR 3



/* Standard set of integer macros  .. */

#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \
                      (((Uint64)((unsigned char*) (s))[1]) << 48) | \
                      (((Uint64)((unsigned char*) (s))[2]) << 40) | \
                      (((Uint64)((unsigned char*) (s))[3]) << 32) | \
                      (((Uint64)((unsigned char*) (s))[4]) << 24) | \
                      (((Uint64)((unsigned char*) (s))[5]) << 16) | \
                      (((Uint64)((unsigned char*) (s))[6]) << 8)  | \
                      (((Uint64)((unsigned char*) (s))[7])))

#define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\
                            ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\
                            ((char*)(s))[2] = (char)((Sint64)(i) >> 40) & 0xff;\
                            ((char*)(s))[3] = (char)((Sint64)(i) >> 32) & 0xff;\
                            ((char*)(s))[4] = (char)((Sint64)(i) >> 24) & 0xff;\
                            ((char*)(s))[5] = (char)((Sint64)(i) >> 16) & 0xff;\
                            ((char*)(s))[6] = (char)((Sint64)(i) >> 8)  & 0xff;\
                            ((char*)(s))[7] = (char)((Sint64)(i))       & 0xff;\
                           } while (0) 

#define get_int32(s) ((((unsigned char*) (s))[0] << 24) | \
                      (((unsigned char*) (s))[1] << 16) | \
                      (((unsigned char*) (s))[2] << 8)  | \
                      (((unsigned char*) (s))[3]))

#define put_int32(i, s) do {((char*)(s))[0] = (char)((i) >> 24) & 0xff;   \
                            ((char*)(s))[1] = (char)((i) >> 16) & 0xff;   \
                            ((char*)(s))[2] = (char)((i) >> 8)  & 0xff;   \
                            ((char*)(s))[3] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \
                      (((unsigned char*) (s))[1] << 8)  | \
                      (((unsigned char*) (s))[2]))

#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff;  \
                            ((char*)(s))[1] = (char)((i) >> 8)  & 0xff;  \
                            ((char*)(s))[2] = (char)(i)         & 0xff;} \
                        while (0)

#define get_int16(s) ((((unsigned char*)  (s))[0] << 8) | \
                      (((unsigned char*)  (s))[1]))


#define put_int16(i, s) do {((char*)(s))[0] = (char)((i) >> 8) & 0xff;  \
                            ((char*)(s))[1] = (char)(i)        & 0xff;} \
                        while (0)

#define get_int8(s) ((((unsigned char*)  (s))[0] ))


#define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0)


/*
 * Use DEBUGF as you would use printf, but use double parentheses:
 *
 *   DEBUGF(("Error: %s\n", error));
 *
 * The output will appear in a special console.
 */

#ifdef DEBUG
void erl_debug(char* format, ...);
void erl_bin_write(unsigned char *, int, int);

#  define DEBUGF(x) erl_debug x
#else
# 1121 "beam/sys.h"
#  define DEBUGF(x)
#endif
# 1123 "beam/sys.h"

#ifdef __WIN32__
#ifdef ARCH_64
#define ERTS_ALLOC_ALIGN_BYTES 16
#define ERTS_SMALL_ABS(Small) _abs64(Small) 
#else
# 1129 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1132 "beam/sys.h"
#else
# 1133 "beam/sys.h"
#define ERTS_ALLOC_ALIGN_BYTES 8
#define ERTS_SMALL_ABS(Small) labs(Small) 
#endif
# 1136 "beam/sys.h"

#ifndef ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC 0
#endif
# 1140 "beam/sys.h"

#ifdef __WIN32__
void call_break_handler(void);
char* last_error(void);
char* win32_errorstr(int);
#endif
# 1146 "beam/sys.h"

/************************************************************************
 * Find out the native filename encoding of the process (look at locale of 
 * Unix processes and just do UTF16 on windows 
 ************************************************************************/
#define ERL_FILENAME_UNKNOWN   (0)
#define ERL_FILENAME_LATIN1    (1)
#define ERL_FILENAME_UTF8      (2)
#define ERL_FILENAME_UTF8_MAC  (3)
#define ERL_FILENAME_WIN_WCHAR (4)

/************************************************************************
 * If a filename in for example list_dir is not in the right encoding, it
 * will be skipped in the resulting list, but depending on a startup setting
 * we will inform the user in different ways. These macros define the
 * different reactions to wrongly coded filenames. In the error case an
 * exception will be thrown by prim_file.
 ************************************************************************/
#define ERL_FILENAME_WARNING_WARNING (0)
#define ERL_FILENAME_WARNING_IGNORE (1)
#define ERL_FILENAME_WARNING_ERROR (2)

/***********************************************************************
 * The user can request a range of character that he/she consider
 * printable. Currently this can be either latin1 or unicode, but
 * in the future a set of ranges, or languages, could be specified.
 ***********************************************************************/
#define ERL_PRINTABLE_CHARACTERS_LATIN1 (0)
#define ERL_PRINTABLE_CHARACTERS_UNICODE (1)

int erts_get_native_filename_encoding(void);
/* The set function is only to be used by erl_init! */
void erts_set_user_requested_filename_encoding(int encoding, int warning);
int erts_get_user_requested_filename_encoding(void);
int erts_get_filename_warning_type(void);
/* This function is called from erl_init. The setting is read by BIF's 
   in io/io_lib. Setting is not atomic. */
void erts_set_printable_characters(int range);
/* Get the setting (ERL_PRINTABLE_CHARACTERS_{LATIN1|UNICODE} */
int erts_get_printable_characters(void);

void erts_init_sys_common_misc(void);

#endif
# 1190 "beam/sys.h"
# 27 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/beam_emu.c"
# 1 "beam/erl_vm.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERL_VM_H__
#define __ERL_VM_H__

/* FORCE_HEAP_FRAGS:
 * Debug provocation to make HAlloc always create heap fragments (if allowed)
 * even if there is room on heap.
 */
/* #define FORCE_HEAP_FRAGS */


#if defined(DEBUG) && !defined(CHECK_FOR_HOLES) && !defined(__WIN32__)
# define CHECK_FOR_HOLES
#endif
# 34 "beam/erl_vm.h"

#define BEAM 1
#define EMULATOR "BEAM"
#define SEQ_TRACE 1

#define CONTEXT_REDS 2000	/* Swap process out after this number */
#define MAX_ARG 255	        /* Max number of arguments allowed */
#define MAX_REG 1024            /* Max number of x(N) registers used */

/* Scheduler stores data for temporary heaps if
   !HEAP_ON_C_STACK. Macros (*TmpHeap*) in global.h selects if we put temporary
   heap data on the C stack or if we use the buffers in the scheduler data. */
#define TMP_HEAP_SIZE 128            /* Number of Eterm in the schedulers
				        small heap for transient heap data */
#define ERL_ARITH_TMP_HEAP_SIZE 4    /* as does erl_arith... */
#define BEAM_EMU_TMP_HEAP_SIZE  2    /* and beam_emu... */

/*
 * The new arithmetic operations need some extra X registers in the register array.
 * so does the gc_bif's (i_gc_bif3 need 3 extra).
 */
#define ERTS_X_REGS_ALLOCATED (MAX_REG+3)

#define INPUT_REDUCTIONS (2 * CONTEXT_REDS)

#define H_DEFAULT_SIZE  233        /* default (heap + stack) min size */
#define VH_DEFAULT_SIZE  32768     /* default virtual (bin) heap min size (words) */

#define CP_SIZE 1

#define ErtsHAllocLockCheck(P) \
  ERTS_SMP_LC_ASSERT(erts_dbg_check_halloc_lock((P)))


#ifdef DEBUG
/*
 * Debug HAlloc that initialize all memory to bad things.
 *
 * To get information about where memory is allocated, insert the two
 * lines below directly after the memset line and use the flag +va.
 *
         VERBOSE(DEBUG_ALLOCATION,("HAlloc @ 0x%08lx (%d) %s:%d\n",     \
                 (unsigned long)HEAP_TOP(p),(sz),__FILE__,__LINE__)),   \
 */
#  ifdef CHECK_FOR_HOLES
#    define INIT_HEAP_MEM(p,sz) erts_set_hole_marker(HEAP_TOP(p), (sz))
#  else
# 81 "beam/erl_vm.h"
#    define INIT_HEAP_MEM(p,sz) memset(HEAP_TOP(p),0x01,(sz)*sizeof(Eterm*))
#  endif
# 83 "beam/erl_vm.h"
#else
# 84 "beam/erl_vm.h"
#  define INIT_HEAP_MEM(p,sz) ((void)0)
#endif /* DEBUG */
# 86 "beam/erl_vm.h"


#ifdef FORCE_HEAP_FRAGS
#  define IS_FORCE_HEAP_FRAGS 1
#else
# 91 "beam/erl_vm.h"
#  define IS_FORCE_HEAP_FRAGS 0
#endif
# 93 "beam/erl_vm.h"

/*
 * Allocate heap memory, first on the ordinary heap;
 * failing that, in a heap fragment.
 */
#define HAllocX(p, sz, xtra)		                              \
  (ASSERT((sz) >= 0),					              \
     ErtsHAllocLockCheck(p),					      \
     (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \
      ? erts_heap_alloc((p),(sz),(xtra))                              \
      : (INIT_HEAP_MEM(p,sz),		                              \
         HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))

#define HAlloc(P, SZ) HAllocX(P,SZ,0)

#define HRelease(p, endp, ptr)					\
  if ((ptr) == (endp)) {					\
     ;								\
  } else if (HEAP_START(p) <= (ptr) && (ptr) < HEAP_TOP(p)) {	\
     HEAP_TOP(p) = (ptr);					\
  } else {							\
     erts_heap_frag_shrink(p, ptr);					\
  }

#define HeapWordsLeft(p) (HEAP_LIMIT(p) - HEAP_TOP(p))

#if defined(DEBUG) || defined(CHECK_FOR_HOLES)
#if HALFWORD_HEAP
# define ERTS_HOLE_MARKER (0xdeadbeef)
#else
# 123 "beam/erl_vm.h"
# define ERTS_HOLE_MARKER (((0xdeadbeef << 24) << 8) | 0xdeadbeef)
#endif
# 125 "beam/erl_vm.h"
#endif
# 126 "beam/erl_vm.h"

/*
 * Allocate heap memory on the ordinary heap, NEVER in a heap
 * segment. The caller must ensure that there is enough words
 * left on the heap before calling HeapOnlyAlloc() (for instance,
 * by testing HeapWordsLeft() and calling the garbage collector
 * if not enough).
 */
#ifdef CHECK_FOR_HOLES
# define HeapOnlyAlloc(p, sz)					\
    (ASSERT((sz) >= 0),					        \
     (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))),	        \
      (erts_set_hole_marker(HEAP_TOP(p), (sz)),			\
       (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))))
#else
# 141 "beam/erl_vm.h"
# define HeapOnlyAlloc(p, sz)					\
    (ASSERT((sz) >= 0),					        \
     (ASSERT(((HEAP_LIMIT(p) - HEAP_TOP(p)) >= (sz))),	        \
      (HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz))))
#endif
# 146 "beam/erl_vm.h"


/*
 * Description for each instruction (defined here because the name and
 * count fields are interesting outside the emulator proper).
 */

typedef struct op_entry {
   char* name;			/* Name of instruction. */
   Uint32 mask[3];		/* Signature mask. */
   int sz;			/* Number of loaded words. */
   char* pack;			/* Instructions for packing engine. */
   char* sign;			/* Signature string. */
   unsigned count;		/* Number of times executed. */
} OpEntry;

extern OpEntry opc[];		/* Description of all instructions. */
extern int num_instructions;	/* Number of instruction in opc[]. */

/* some constants for various table sizes etc */

#define ATOM_TEXT_SIZE  32768	/* Increment for allocating atom text space */

#define ITIME 100		/* Number of milliseconds per clock tick    */
#define MAX_PORT_LINK 8		/* Maximum number of links to a port        */

extern int H_MIN_SIZE;		/* minimum (heap + stack) */
extern int BIN_VH_MIN_SIZE;	/* minimum virtual (bin) heap */

extern int erts_atom_table_size;/* Atom table size */
extern int erts_pd_initial_size;/* Initial Process dictionary table size */

#define ORIG_CREATION 0
#define INTERNAL_CREATION 255

/* macros for extracting bytes from uint16's */

#define hi_byte(a) ((a) >> 8) 
#define lo_byte(a) ((a) & 255) 

/* macros for combining bytes */

#define make_16(x, y) (((x) << 8) | (y))
#define make_24(x,y,z) (((x) << 16) | ((y) << 8) | (z))
#define make_32(x3,x2,x1,x0) (((x3)<<24) | ((x2)<<16) | ((x1)<<8) | (x0))

#define make_signed_24(x,y,z) ((sint32) (((x) << 24) | ((y) << 16) | ((z) << 8)) >> 8)
#define make_signed_32(x3,x2,x1,x0) ((sint32) (((x3) << 24) | ((x2) << 16) | ((x1) << 8) | (x0)))

#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 195 "beam/erl_vm.h"
# 196 "beam/erl_vm.h"

#endif	/* __ERL_VM_H__ */
# 198 "beam/erl_vm.h"
# 28 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "global.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/beam_emu.c"
# 1 "beam/global.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __GLOBAL_H__
#define __GLOBAL_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/global.h"
# 25 "beam/global.h"
#if 0 /* expanded by -frewrite-includes */
#include <stddef.h> /* offsetof() */
#endif /* expanded by -frewrite-includes */
# 25 "beam/global.h"
# 26 "beam/global.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc.h"
#endif /* expanded by -frewrite-includes */
# 26 "beam/global.h"
# 1 "beam/erl_alloc.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2002-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_ALLOC_H__
#define ERL_ALLOC_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc_types.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_alloc.h"
# 1 "armv6-portbld-freebsd11.0/opt/smp/erl_alloc_types.h" 1
/*
 * -----------------------------------------------------------------------
 *
 * NOTE: Do *not* edit this file; instead, edit 'erl_alloc.types' and
 *       build again! This file was automatically generated by
 *       'make_alloc_types' on Thu May 12 14:30:00 2016.
 *
 * -----------------------------------------------------------------------
 *
 * 
 * Copyright Ericsson AB 2016. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 */

#ifndef ERL_ALLOC_TYPES_H__
#define ERL_ALLOC_TYPES_H__


/* --- Allocator numbers -------------------------------------------------- */

#define ERTS_ALC_A_INVALID (0)

#define ERTS_ALC_A_MIN (1)

#define ERTS_ALC_A_SYSTEM (1)
#define ERTS_ALC_A_TEMPORARY (2)
#define ERTS_ALC_A_SHORT_LIVED (3)
#define ERTS_ALC_A_STANDARD (4)
#define ERTS_ALC_A_LONG_LIVED (5)
#define ERTS_ALC_A_EHEAP (6)
#define ERTS_ALC_A_ETS (7)
#define ERTS_ALC_A_FIXED_SIZE (8)
#define ERTS_ALC_A_BINARY (9)
#define ERTS_ALC_A_DRIVER (10)
#define ERTS_ALC_A_TEST (11)

#define ERTS_ALC_A_MAX (11)


/* --- Class numbers ------------------------------------------------------ */

#define ERTS_ALC_C_INVALID (0)

#define ERTS_ALC_C_MIN (1)

#define ERTS_ALC_C_ATOM  (1)
#define ERTS_ALC_C_BINARIES  (2)
#define ERTS_ALC_C_CODE  (3)
#define ERTS_ALC_C_ETS  (4)
#define ERTS_ALC_C_PROCESSES  (5)
#define ERTS_ALC_C_SYSTEM  (6)

#define ERTS_ALC_C_MAX (6)


/* --- Type number intervals ---------------------------------------------- */

#define ERTS_ALC_N_INVALID (0)

#define ERTS_ALC_N_MIN (1)

/* Type numbers used for sys_alloc */
#define ERTS_ALC_N_MIN_A_SYSTEM (1)
#define ERTS_ALC_N_MAX_A_SYSTEM (5)

/* Type numbers used for temp_alloc */
#define ERTS_ALC_N_MIN_A_TEMPORARY (6)
#define ERTS_ALC_N_MAX_A_TEMPORARY (25)

/* Type numbers used for sl_alloc */
#define ERTS_ALC_N_MIN_A_SHORT_LIVED (26)
#define ERTS_ALC_N_MAX_A_SHORT_LIVED (70)

/* Type numbers used for std_alloc */
#define ERTS_ALC_N_MIN_A_STANDARD (71)
#define ERTS_ALC_N_MAX_A_STANDARD (109)

/* Type numbers used for ll_alloc */
#define ERTS_ALC_N_MIN_A_LONG_LIVED (110)
#define ERTS_ALC_N_MAX_A_LONG_LIVED (155)

/* Type numbers used for eheap_alloc */
#define ERTS_ALC_N_MIN_A_EHEAP (156)
#define ERTS_ALC_N_MAX_A_EHEAP (159)

/* Type numbers used for ets_alloc */
#define ERTS_ALC_N_MIN_A_ETS (160)
#define ERTS_ALC_N_MAX_A_ETS (168)

/* Type numbers used for fix_alloc */
#define ERTS_ALC_N_MIN_A_FIXED_SIZE (169)
#define ERTS_ALC_N_MAX_A_FIXED_SIZE (178)

/* Type numbers used for binary_alloc */
#define ERTS_ALC_N_MIN_A_BINARY (179)
#define ERTS_ALC_N_MAX_A_BINARY (180)

/* Type numbers used for driver_alloc */
#define ERTS_ALC_N_MIN_A_DRIVER (181)
#define ERTS_ALC_N_MAX_A_DRIVER (194)

/* Type numbers used for test_alloc */
#define ERTS_ALC_N_MIN_A_TEST (195)
#define ERTS_ALC_N_MAX_A_TEST (195)

#define ERTS_ALC_N_MAX (195)


/* --- Multi thread use of allocators -------------------------------------- */

#define ERTS_ALC_MTA_SYSTEM (1)
#define ERTS_ALC_MTA_TEMPORARY (1)
#define ERTS_ALC_MTA_SHORT_LIVED (1)
#define ERTS_ALC_MTA_STANDARD (1)
#define ERTS_ALC_MTA_LONG_LIVED (1)
#define ERTS_ALC_MTA_EHEAP (1)
#define ERTS_ALC_MTA_ETS (1)
#define ERTS_ALC_MTA_FIXED_SIZE (1)
#define ERTS_ALC_MTA_BINARY (1)
#define ERTS_ALC_MTA_DRIVER (1)
#define ERTS_ALC_MTA_TEST (1)


/* --- Types --------------------------------------------------------------- */

typedef Uint32 ErtsAlcType_t; /* The type used for memory types */

#define ERTS_ALC_T_INVALID (0)

#define ERTS_ALC_T_UNDEF (225)
#define ERTS_ALC_T_NBIF_TABLE (353)
#define ERTS_ALC_T_INFO_DSBUF (481)
#define ERTS_ALC_T_HIPE (609)
#define ERTS_ALC_T_PUTENV_STR (737)
#define ERTS_ALC_T_TMP_HEAP (850)
#define ERTS_ALC_T_MSG_ROOTS (978)
#define ERTS_ALC_T_ROOTSET (1106)
#define ERTS_ALC_T_LOADER_TMP (1202)
#define ERTS_ALC_T_NC_TMP (1378)
#define ERTS_ALC_T_TMP (1506)
#define ERTS_ALC_T_DCTRL_BUF (1634)
#define ERTS_ALC_T_TMP_DIST_BUF (1762)
#define ERTS_ALC_T_ESTACK (1890)
#define ERTS_ALC_T_DB_TMP (1986)
#define ERTS_ALC_T_DB_MC_STK (2114)
#define ERTS_ALC_T_DB_MS_CMPL_HEAP (2242)
#define ERTS_ALC_T_LOGGER_DSBUF (2402)
#define ERTS_ALC_T_TMP_DSBUF (2530)
#define ERTS_ALC_T_DDLL_TMP_BUF (2658)
#define ERTS_ALC_T_RE_TMP_BUF (2786)
#define ERTS_ALC_T_TEMP_TERM (2914)
#define ERTS_ALC_T_SYS_READ_BUF (3042)
#define ERTS_ALC_T_ENVIRONMENT (3170)
#define ERTS_ALC_T_SYS_WRITE_BUF (3298)
#define ERTS_ALC_T_PEND_SUSPEND (3411)
#define ERTS_ALC_T_PROC_LIST (3539)
#define ERTS_ALC_T_SAVED_ESTACK (3667)
#define ERTS_ALC_T_PREPARED_CODE (3763)
#define ERTS_ALC_T_TIMER_REQUEST (3923)
#define ERTS_ALC_T_BTM_YIELD_STATE (4051)
#define ERTS_ALC_T_DB_FIXATION (4163)
#define ERTS_ALC_T_DB_FIX_DEL (4291)
#define ERTS_ALC_T_DB_MS_RUN_HEAP (4419)
#define ERTS_ALC_T_DB_PROC_CLEANUP (4547)
#define ERTS_ALC_T_PORT_TASK (4707)
#define ERTS_ALC_T_PT_HNDL_LIST (4835)
#define ERTS_ALC_T_MISC_OP_LIST (4963)
#define ERTS_ALC_T_PORT_NAMES (5091)
#define ERTS_ALC_T_PTAB_LIST_DEL (5203)
#define ERTS_ALC_T_PTAB_LIST_CNKI (5331)
#define ERTS_ALC_T_PTAB_LIST_PIDS (5459)
#define ERTS_ALC_T_RE_SUBJECT (5603)
#define ERTS_ALC_T_RE_STACK (5731)
#define ERTS_ALC_T_UNICODE_BUFFER (5859)
#define ERTS_ALC_T_BINARY_BUFFER (5987)
#define ERTS_ALC_T_TMP_CPU_IDS (6115)
#define ERTS_ALC_T_EXT_TERM_DATA (6227)
#define ERTS_ALC_T_CODE_IX_LOCK_Q (6371)
#define ERTS_ALC_T_BUSY_CALLER_TAB (6499)
#define ERTS_ALC_T_BUSY_CALLER (6627)
#define ERTS_ALC_T_PROC_SYS_TSK (6739)
#define ERTS_ALC_T_PROC_SYS_TSK_QS (6867)
#define ERTS_ALC_T_NEW_TIME_OFFSET (7011)
#define ERTS_ALC_T_IOB_REQ (7139)
#define ERTS_ALC_T_MISC_AUX_WORK (7267)
#define ERTS_ALC_T_THR_Q_SL (7395)
#define ERTS_ALC_T_ASYNC (7523)
#define ERTS_ALC_T_XPORTS_LIST (7651)
#define ERTS_ALC_T_T_THR_PRGR_DATA (7779)
#define ERTS_ALC_T_ETHR_SL (7907)
#define ERTS_ALC_T_SYS_MSG_Q (8019)
#define ERTS_ALC_T_SL_MPATHS (8163)
#define ERTS_ALC_T_AINFO_REQ (8291)
#define ERTS_ALC_T_SCHED_WTIME_REQ (8419)
#define ERTS_ALC_T_GC_INFO_REQ (8547)
#define ERTS_ALC_T_SYS_CHECK_REQ (8675)
#define ERTS_ALC_T_FD_LIST (8803)
#define ERTS_ALC_T_ACTIVE_FD_ARR (8931)
#define ERTS_ALC_T_POLLSET_UPDREQ (9059)
#define ERTS_ALC_T_REG_PROC (9172)
#define ERTS_ALC_T_LINK_LH (9300)
#define ERTS_ALC_T_SUSPEND_MON (9428)
#define ERTS_ALC_T_REG_TABLE (9572)
#define ERTS_ALC_T_FUN_TABLE (9652)
#define ERTS_ALC_T_DIST_TABLE (9828)
#define ERTS_ALC_T_NODE_TABLE (9956)
#define ERTS_ALC_T_MODULE_REFS (10036)
#define ERTS_ALC_T_DCACHE (10212)
#define ERTS_ALC_T_DIST_ENTRY (10340)
#define ERTS_ALC_T_NODE_ENTRY (10468)
#define ERTS_ALC_T_ARG_REG (10580)
#define ERTS_ALC_T_PROC_DICT (10708)
#define ERTS_ALC_T_CALLS_BUF (10836)
#define ERTS_ALC_T_BPD (10980)
#define ERTS_ALC_T_LINEBUF (11108)
#define ERTS_ALC_T_IOQ (11236)
#define ERTS_ALC_T_BITS_BUF (11364)
#define ERTS_ALC_T_DB_NTAB_ENT (11460)
#define ERTS_ALC_T_DDLL_HANDLE (11620)
#define ERTS_ALC_T_NODES_MON (11732)
#define ERTS_ALC_T_RE_HEAP (11876)
#define ERTS_ALC_T_PSD (11988)
#define ERTS_ALC_T_PRTSD (12132)
#define ERTS_ALC_T_ZLIB (12260)
#define ERTS_ALC_T_THR_Q_EL (12388)
#define ERTS_ALC_T_THR_Q (12516)
#define ERTS_ALC_T_PORT_LOCK (12644)
#define ERTS_ALC_T_DRIVER_LOCK (12772)
#define ERTS_ALC_T_ETHR_STD (12900)
#define ERTS_ALC_T_DDLL_PROCESS (13028)
#define ERTS_ALC_T_MONITOR_LH (13140)
#define ERTS_ALC_T_NLINK_LH (13268)
#define ERTS_ALC_T_DB_HEIR_DATA (13380)
#define ERTS_ALC_T_NIF_TRAP_EXPORT (13492)
#define ERTS_ALC_T_PORT_DATA_HEAP (13668)
#define ERTS_ALC_T_FD_ENTRY_BUF (13796)
#define ERTS_ALC_T_PRT_REP_EXIT (13924)
#define ERTS_ALC_T_SYS_BLOCKING (14052)
#define ERTS_ALC_T_ATOM (14101)
#define ERTS_ALC_T_MODULE (14261)
#define ERTS_ALC_T_FUN_ENTRY (14389)
#define ERTS_ALC_T_ATOM_TXT (14485)
#define ERTS_ALC_T_TIMER_SERVICE (14693)
#define ERTS_ALC_T_ATOM_TABLE (14741)
#define ERTS_ALC_T_EXPORT_TABLE (14901)
#define ERTS_ALC_T_MODULE_TABLE (15029)
#define ERTS_ALC_T_TAINT (15157)
#define ERTS_ALC_T_PROC_TABLE (15317)
#define ERTS_ALC_T_PORT_TABLE (15461)
#define ERTS_ALC_T_TIMER_WHEEL (15589)
#define ERTS_ALC_T_ASYNC_DATA (15717)
#define ERTS_ALC_T_DB_TABLES (15813)
#define ERTS_ALC_T_INSTR_INFO (15973)
#define ERTS_ALC_T_SCHDLR_SLP_INFO (16101)
#define ERTS_ALC_T_RUNQS (16229)
#define ERTS_ALC_T_DDLL_ERRCODES (16357)
#define ERTS_ALC_T_PRE_ALLOC_DATA (16485)
#define ERTS_ALC_T_CPUDATA (16613)
#define ERTS_ALC_T_CPU_GRPS_MAP (16741)
#define ERTS_ALC_T_AUX_WORK_TMO (16869)
#define ERTS_ALC_T_MISC_AUX_WORK_Q (16997)
#define ERTS_ALC_T_PROC_INTERVAL (17125)
#define ERTS_ALC_T_THR_Q_LL (17253)
#define ERTS_ALC_T_PROC_LCK_WTR (17381)
#define ERTS_ALC_T_RUNQ_BLNS (17509)
#define ERTS_ALC_T_THR_PRGR_IDATA (17637)
#define ERTS_ALC_T_THR_PRGR_DATA (17765)
#define ERTS_ALC_T_ETHR_LL (17893)
#define ERTS_ALC_T_FP_EXCEPTION (18021)
#define ERTS_ALC_T_LL_MPATHS (18149)
#define ERTS_ALC_T_CODE (18229)
#define ERTS_ALC_T_DB_MS_PSDO_PROC (18373)
#define ERTS_ALC_T_SCHDLR_DATA (18533)
#define ERTS_ALC_T_LL_TEMP_TERM (18661)
#define ERTS_ALC_T_EXPORT (18741)
#define ERTS_ALC_T_DRV_TAB (18917)
#define ERTS_ALC_T_DRV_EV_STATE (19045)
#define ERTS_ALC_T_POLLSET (19173)
#define ERTS_ALC_T_POLL_FDS (19301)
#define ERTS_ALC_T_POLL_RES_EVS (19429)
#define ERTS_ALC_T_FD_STATUS (19557)
#define ERTS_ALC_T_SELECT_FDS (19685)
#define ERTS_ALC_T_FD_TAB (19813)
#define ERTS_ALC_T_CS_PROG_PATH (19941)
#define ERTS_ALC_T_BEAM_REGISTER (20054)
#define ERTS_ALC_T_HEAP (20182)
#define ERTS_ALC_T_OLD_HEAP (20310)
#define ERTS_ALC_T_HEAP_FRAG (20438)
#define ERTS_ALC_T_DB_TABLE (20551)
#define ERTS_ALC_T_DB_SEG (20679)
#define ERTS_ALC_T_DB_SEG_TAB (20807)
#define ERTS_ALC_T_DB_STK (20935)
#define ERTS_ALC_T_DB_TRANS_TAB (21063)
#define ERTS_ALC_T_DB_SEL_LIST (21191)
#define ERTS_ALC_T_DB_DMC_ERROR (21319)
#define ERTS_ALC_T_DB_DMC_ERR_INFO (21447)
#define ERTS_ALC_T_DB_TERM (21575)
#define ERTS_ALC_T_PROC (21720)
#define ERTS_ALC_T_MSG_REF (21848)
#define ERTS_ALC_T_LL_PTIMER (21976)
#define ERTS_ALC_T_HL_PTIMER (22104)
#define ERTS_ALC_T_BIF_TIMER (22232)
#define ERTS_ALC_T_THR_Q_EL_SL (22376)
#define ERTS_ALC_T_MONITOR_SH (22488)
#define ERTS_ALC_T_NLINK_SH (22616)
#define ERTS_ALC_T_DRV_EV_D_STATE (22760)
#define ERTS_ALC_T_DRV_SEL_D_STATE (22888)
#define ERTS_ALC_T_DRV_BINARY (22953)
#define ERTS_ALC_T_BINARY (23081)
#define ERTS_ALC_T_PORT (23274)
#define ERTS_ALC_T_DRV (23402)
#define ERTS_ALC_T_DRIVER (23530)
#define ERTS_ALC_T_DRV_CMD_DATA (23658)
#define ERTS_ALC_T_DRV_CTRL_DATA (23786)
#define ERTS_ALC_T_DRV_CALL_DATA (23914)
#define ERTS_ALC_T_NIF (24042)
#define ERTS_ALC_T_PORT_DATA_LOCK (24170)
#define ERTS_ALC_T_DRV_THR_OPTS (24298)
#define ERTS_ALC_T_DRV_TID (24426)
#define ERTS_ALC_T_DRV_MTX (24554)
#define ERTS_ALC_T_DRV_CND (24682)
#define ERTS_ALC_T_DRV_RWLCK (24810)
#define ERTS_ALC_T_DRV_TSD (24938)
#define ERTS_ALC_T_TEST (25067)


/* --- Field sizes, masks, and shifts -------------------------------------- */

#define ERTS_ALC_A_BITS (4)
#define ERTS_ALC_C_BITS (3)
#define ERTS_ALC_N_BITS (8)
#define ERTS_ALC_T_BITS (15)

#define ERTS_ALC_A_MASK (15)
#define ERTS_ALC_C_MASK (7)
#define ERTS_ALC_N_MASK (255)
#define ERTS_ALC_T_MASK (32767)

#define ERTS_ALC_A_SHIFT (0)
#define ERTS_ALC_C_SHIFT (4)
#define ERTS_ALC_N_SHIFT (7)


/* --- Mappings ------------------------------------------------------------ */

/* type -> type number */
#define ERTS_ALC_T2N(T) (((T) >> ERTS_ALC_N_SHIFT) & ERTS_ALC_N_MASK)

/* type -> allocator number */
#define ERTS_ALC_T2A(T) (((T) >> ERTS_ALC_A_SHIFT) & ERTS_ALC_A_MASK)

/* type -> class number */
#define ERTS_ALC_T2C(T) (((T) >> ERTS_ALC_C_SHIFT) & ERTS_ALC_C_MASK)

/* type number -> type */
#define ERTS_ALC_N2T(N) (erts_alc_n2t[(N)])

/* type number -> type description */
#define ERTS_ALC_N2TD(N) (erts_alc_n2td[(N)])

/* type -> type description */
#define ERTS_ALC_T2TD(T) (ERTS_ALC_N2TD(ERTS_ALC_T2N((T))))

/* class number -> class description */
#define ERTS_ALC_C2CD(C) (erts_alc_c2cd[(C)])

/* allocator number -> allocator description */
#define ERTS_ALC_A2AD(A) (erts_alc_a2ad[(A)])

extern const ErtsAlcType_t erts_alc_n2t[];
extern const char *erts_alc_n2td[];
extern const char *erts_alc_c2cd[];
extern const char *erts_alc_a2ad[];

#ifdef ERTS_ALC_INTERNAL__

const ErtsAlcType_t erts_alc_n2t[] = {
  ERTS_ALC_T_INVALID,
  ERTS_ALC_T_UNDEF,
  ERTS_ALC_T_NBIF_TABLE,
  ERTS_ALC_T_INFO_DSBUF,
  ERTS_ALC_T_HIPE,
  ERTS_ALC_T_PUTENV_STR,
  ERTS_ALC_T_TMP_HEAP,
  ERTS_ALC_T_MSG_ROOTS,
  ERTS_ALC_T_ROOTSET,
  ERTS_ALC_T_LOADER_TMP,
  ERTS_ALC_T_NC_TMP,
  ERTS_ALC_T_TMP,
  ERTS_ALC_T_DCTRL_BUF,
  ERTS_ALC_T_TMP_DIST_BUF,
  ERTS_ALC_T_ESTACK,
  ERTS_ALC_T_DB_TMP,
  ERTS_ALC_T_DB_MC_STK,
  ERTS_ALC_T_DB_MS_CMPL_HEAP,
  ERTS_ALC_T_LOGGER_DSBUF,
  ERTS_ALC_T_TMP_DSBUF,
  ERTS_ALC_T_DDLL_TMP_BUF,
  ERTS_ALC_T_RE_TMP_BUF,
  ERTS_ALC_T_TEMP_TERM,
  ERTS_ALC_T_SYS_READ_BUF,
  ERTS_ALC_T_ENVIRONMENT,
  ERTS_ALC_T_SYS_WRITE_BUF,
  ERTS_ALC_T_PEND_SUSPEND,
  ERTS_ALC_T_PROC_LIST,
  ERTS_ALC_T_SAVED_ESTACK,
  ERTS_ALC_T_PREPARED_CODE,
  ERTS_ALC_T_TIMER_REQUEST,
  ERTS_ALC_T_BTM_YIELD_STATE,
  ERTS_ALC_T_DB_FIXATION,
  ERTS_ALC_T_DB_FIX_DEL,
  ERTS_ALC_T_DB_MS_RUN_HEAP,
  ERTS_ALC_T_DB_PROC_CLEANUP,
  ERTS_ALC_T_PORT_TASK,
  ERTS_ALC_T_PT_HNDL_LIST,
  ERTS_ALC_T_MISC_OP_LIST,
  ERTS_ALC_T_PORT_NAMES,
  ERTS_ALC_T_PTAB_LIST_DEL,
  ERTS_ALC_T_PTAB_LIST_CNKI,
  ERTS_ALC_T_PTAB_LIST_PIDS,
  ERTS_ALC_T_RE_SUBJECT,
  ERTS_ALC_T_RE_STACK,
  ERTS_ALC_T_UNICODE_BUFFER,
  ERTS_ALC_T_BINARY_BUFFER,
  ERTS_ALC_T_TMP_CPU_IDS,
  ERTS_ALC_T_EXT_TERM_DATA,
  ERTS_ALC_T_CODE_IX_LOCK_Q,
  ERTS_ALC_T_BUSY_CALLER_TAB,
  ERTS_ALC_T_BUSY_CALLER,
  ERTS_ALC_T_PROC_SYS_TSK,
  ERTS_ALC_T_PROC_SYS_TSK_QS,
  ERTS_ALC_T_NEW_TIME_OFFSET,
  ERTS_ALC_T_IOB_REQ,
  ERTS_ALC_T_MISC_AUX_WORK,
  ERTS_ALC_T_THR_Q_SL,
  ERTS_ALC_T_ASYNC,
  ERTS_ALC_T_XPORTS_LIST,
  ERTS_ALC_T_T_THR_PRGR_DATA,
  ERTS_ALC_T_ETHR_SL,
  ERTS_ALC_T_SYS_MSG_Q,
  ERTS_ALC_T_SL_MPATHS,
  ERTS_ALC_T_AINFO_REQ,
  ERTS_ALC_T_SCHED_WTIME_REQ,
  ERTS_ALC_T_GC_INFO_REQ,
  ERTS_ALC_T_SYS_CHECK_REQ,
  ERTS_ALC_T_FD_LIST,
  ERTS_ALC_T_ACTIVE_FD_ARR,
  ERTS_ALC_T_POLLSET_UPDREQ,
  ERTS_ALC_T_REG_PROC,
  ERTS_ALC_T_LINK_LH,
  ERTS_ALC_T_SUSPEND_MON,
  ERTS_ALC_T_REG_TABLE,
  ERTS_ALC_T_FUN_TABLE,
  ERTS_ALC_T_DIST_TABLE,
  ERTS_ALC_T_NODE_TABLE,
  ERTS_ALC_T_MODULE_REFS,
  ERTS_ALC_T_DCACHE,
  ERTS_ALC_T_DIST_ENTRY,
  ERTS_ALC_T_NODE_ENTRY,
  ERTS_ALC_T_ARG_REG,
  ERTS_ALC_T_PROC_DICT,
  ERTS_ALC_T_CALLS_BUF,
  ERTS_ALC_T_BPD,
  ERTS_ALC_T_LINEBUF,
  ERTS_ALC_T_IOQ,
  ERTS_ALC_T_BITS_BUF,
  ERTS_ALC_T_DB_NTAB_ENT,
  ERTS_ALC_T_DDLL_HANDLE,
  ERTS_ALC_T_NODES_MON,
  ERTS_ALC_T_RE_HEAP,
  ERTS_ALC_T_PSD,
  ERTS_ALC_T_PRTSD,
  ERTS_ALC_T_ZLIB,
  ERTS_ALC_T_THR_Q_EL,
  ERTS_ALC_T_THR_Q,
  ERTS_ALC_T_PORT_LOCK,
  ERTS_ALC_T_DRIVER_LOCK,
  ERTS_ALC_T_ETHR_STD,
  ERTS_ALC_T_DDLL_PROCESS,
  ERTS_ALC_T_MONITOR_LH,
  ERTS_ALC_T_NLINK_LH,
  ERTS_ALC_T_DB_HEIR_DATA,
  ERTS_ALC_T_NIF_TRAP_EXPORT,
  ERTS_ALC_T_PORT_DATA_HEAP,
  ERTS_ALC_T_FD_ENTRY_BUF,
  ERTS_ALC_T_PRT_REP_EXIT,
  ERTS_ALC_T_SYS_BLOCKING,
  ERTS_ALC_T_ATOM,
  ERTS_ALC_T_MODULE,
  ERTS_ALC_T_FUN_ENTRY,
  ERTS_ALC_T_ATOM_TXT,
  ERTS_ALC_T_TIMER_SERVICE,
  ERTS_ALC_T_ATOM_TABLE,
  ERTS_ALC_T_EXPORT_TABLE,
  ERTS_ALC_T_MODULE_TABLE,
  ERTS_ALC_T_TAINT,
  ERTS_ALC_T_PROC_TABLE,
  ERTS_ALC_T_PORT_TABLE,
  ERTS_ALC_T_TIMER_WHEEL,
  ERTS_ALC_T_ASYNC_DATA,
  ERTS_ALC_T_DB_TABLES,
  ERTS_ALC_T_INSTR_INFO,
  ERTS_ALC_T_SCHDLR_SLP_INFO,
  ERTS_ALC_T_RUNQS,
  ERTS_ALC_T_DDLL_ERRCODES,
  ERTS_ALC_T_PRE_ALLOC_DATA,
  ERTS_ALC_T_CPUDATA,
  ERTS_ALC_T_CPU_GRPS_MAP,
  ERTS_ALC_T_AUX_WORK_TMO,
  ERTS_ALC_T_MISC_AUX_WORK_Q,
  ERTS_ALC_T_PROC_INTERVAL,
  ERTS_ALC_T_THR_Q_LL,
  ERTS_ALC_T_PROC_LCK_WTR,
  ERTS_ALC_T_RUNQ_BLNS,
  ERTS_ALC_T_THR_PRGR_IDATA,
  ERTS_ALC_T_THR_PRGR_DATA,
  ERTS_ALC_T_ETHR_LL,
  ERTS_ALC_T_FP_EXCEPTION,
  ERTS_ALC_T_LL_MPATHS,
  ERTS_ALC_T_CODE,
  ERTS_ALC_T_DB_MS_PSDO_PROC,
  ERTS_ALC_T_SCHDLR_DATA,
  ERTS_ALC_T_LL_TEMP_TERM,
  ERTS_ALC_T_EXPORT,
  ERTS_ALC_T_DRV_TAB,
  ERTS_ALC_T_DRV_EV_STATE,
  ERTS_ALC_T_POLLSET,
  ERTS_ALC_T_POLL_FDS,
  ERTS_ALC_T_POLL_RES_EVS,
  ERTS_ALC_T_FD_STATUS,
  ERTS_ALC_T_SELECT_FDS,
  ERTS_ALC_T_FD_TAB,
  ERTS_ALC_T_CS_PROG_PATH,
  ERTS_ALC_T_BEAM_REGISTER,
  ERTS_ALC_T_HEAP,
  ERTS_ALC_T_OLD_HEAP,
  ERTS_ALC_T_HEAP_FRAG,
  ERTS_ALC_T_DB_TABLE,
  ERTS_ALC_T_DB_SEG,
  ERTS_ALC_T_DB_SEG_TAB,
  ERTS_ALC_T_DB_STK,
  ERTS_ALC_T_DB_TRANS_TAB,
  ERTS_ALC_T_DB_SEL_LIST,
  ERTS_ALC_T_DB_DMC_ERROR,
  ERTS_ALC_T_DB_DMC_ERR_INFO,
  ERTS_ALC_T_DB_TERM,
  ERTS_ALC_T_PROC,
  ERTS_ALC_T_MSG_REF,
  ERTS_ALC_T_LL_PTIMER,
  ERTS_ALC_T_HL_PTIMER,
  ERTS_ALC_T_BIF_TIMER,
  ERTS_ALC_T_THR_Q_EL_SL,
  ERTS_ALC_T_MONITOR_SH,
  ERTS_ALC_T_NLINK_SH,
  ERTS_ALC_T_DRV_EV_D_STATE,
  ERTS_ALC_T_DRV_SEL_D_STATE,
  ERTS_ALC_T_DRV_BINARY,
  ERTS_ALC_T_BINARY,
  ERTS_ALC_T_PORT,
  ERTS_ALC_T_DRV,
  ERTS_ALC_T_DRIVER,
  ERTS_ALC_T_DRV_CMD_DATA,
  ERTS_ALC_T_DRV_CTRL_DATA,
  ERTS_ALC_T_DRV_CALL_DATA,
  ERTS_ALC_T_NIF,
  ERTS_ALC_T_PORT_DATA_LOCK,
  ERTS_ALC_T_DRV_THR_OPTS,
  ERTS_ALC_T_DRV_TID,
  ERTS_ALC_T_DRV_MTX,
  ERTS_ALC_T_DRV_CND,
  ERTS_ALC_T_DRV_RWLCK,
  ERTS_ALC_T_DRV_TSD,
  ERTS_ALC_T_TEST,
  ERTS_ALC_T_INVALID
};

const char *erts_alc_n2td[] = {
  "invalid_type",
  "undefined",
  "nbif_tab",
  "info_dsbuf",
  "hipe_data",
  "putenv_string",
  "tmp_heap",
  "msg_roots",
  "root_set",
  "loader_tmp",
  "nc_tmp",
  "tmp",
  "dctrl_buf",
  "tmp_dist_buf",
  "estack",
  "db_tmp",
  "db_mc_stack",
  "db_match_spec_cmpl_heap",
  "logger_dsbuf",
  "tmp_dsbuf",
  "ddll_tmp_buf",
  "re_tmp_buf",
  "temp_term",
  "sys_read_buf",
  "environment",
  "sys_write_buf",
  "pending_suspend",
  "proc_list",
  "saved_estack",
  "prepared_code",
  "timer_request",
  "btm_yield_state",
  "db_fixation",
  "fixed_del",
  "db_match_spec_run_heap",
  "db_proc_cleanup_state",
  "port_task",
  "port_task_handle_list",
  "misc_op_list",
  "port_names",
  "ptab_list_deleted_el",
  "ptab_list_chunk_info",
  "ptab_list_pids",
  "re_subject",
  "re_stack",
  "unicode_buffer",
  "binary_buffer",
  "tmp_cpu_ids",
  "external_term_data",
  "code_ix_lock_q",
  "busy_caller_table",
  "busy_caller",
  "proc_sys_task",
  "proc_sys_task_queues",
  "new_time_offset",
  "io_bytes_request",
  "misc_aux_work",
  "short_lived_thr_queue",
  "async",
  "extra_port_list",
  "temp_thr_prgr_data",
  "ethread_short_lived",
  "system_messages_queue",
  "sl_migration_paths",
  "alloc_info_request",
  "sched_wall_time_request",
  "gc_info_request",
  "system_check_request",
  "fd_list",
  "active_fd_array",
  "pollset_update_req",
  "reg_proc",
  "link_lh",
  "suspend_monitor",
  "reg_tab",
  "fun_tab",
  "dist_tab",
  "node_tab",
  "module_refs",
  "dcache",
  "dist_entry",
  "node_entry",
  "arg_reg",
  "proc_dict",
  "calls_buf",
  "bpd",
  "line_buf",
  "io_queue",
  "bits_buf",
  "db_named_table_entry",
  "ddll_handle",
  "nodes_monitor",
  "re_heap",
  "process_specific_data",
  "port_specific_data",
  "zlib",
  "thr_q_element",
  "thr_queue",
  "port_lock",
  "driver_lock",
  "ethread_standard",
  "ddll_processes",
  "monitor_lh",
  "nlink_lh",
  "db_heir_data",
  "nif_trap_export_entry",
  "port_data_heap",
  "fd_entry_buf",
  "port_report_exit",
  "sys_blocking",
  "atom_entry",
  "module_entry",
  "fun_entry",
  "atom_text",
  "timer_service",
  "atom_tab",
  "export_tab",
  "module_tab",
  "taint_list",
  "proc_tab",
  "port_tab",
  "timer_wheel",
  "internal_async_data",
  "db_tabs",
  "instr_info",
  "scheduler_sleep_info",
  "run_queues",
  "ddll_errcodes",
  "pre_alloc_data",
  "cpu_data",
  "cpu_groups_map",
  "aux_work_timeouts",
  "misc_aux_work_q",
  "process_interval",
  "long_lived_thr_queue",
  "proc_lock_waiter",
  "run_queue_balancing",
  "thr_prgr_internal_data",
  "thr_prgr_data",
  "ethread_long_lived",
  "fp_exception",
  "ll_migration_paths",
  "code",
  "db_match_pseudo_proc",
  "scheduler_data",
  "ll_temp_term",
  "export_entry",
  "drv_tab",
  "driver_event_state",
  "pollset",
  "poll_fds",
  "poll_result_events",
  "fd_status",
  "select_fds",
  "fd_tab",
  "cs_prog_path",
  "beam_register",
  "heap",
  "old_heap",
  "heap_frag",
  "db_tab",
  "db_segment",
  "db_segment_tab",
  "db_stack",
  "db_trans_tab",
  "db_select_list",
  "db_dmc_error",
  "db_dmc_error_info",
  "db_term",
  "proc",
  "msg_ref",
  "ll_ptimer",
  "hl_ptimer",
  "bif_timer",
  "sl_thr_q_element",
  "monitor_sh",
  "nlink_sh",
  "driver_event_data_state",
  "driver_select_data_state",
  "drv_binary",
  "binary",
  "port",
  "drv_internal",
  "driver",
  "driver_command_data",
  "driver_control_data",
  "driver_call_data",
  "nif_internal",
  "port_data_lock",
  "driver_thread_opts",
  "driver_tid",
  "driver_mutex",
  "driver_cond",
  "driver_rwlock",
  "driver_tsd",
  "testing",
  NULL
};

const char *erts_alc_c2cd[] = {
  "invalid_class",
  "atom_data",
  "binary_data",
  "code_data",
  "ets_data",
  "process_data",
  "system_data",
  NULL
};

const char *erts_alc_a2ad[] = {
  "invalid_allocator",
  "sys_alloc",
  "temp_alloc",
  "sl_alloc",
  "std_alloc",
  "ll_alloc",
  "eheap_alloc",
  "ets_alloc",
  "fix_alloc",
  "binary_alloc",
  "driver_alloc",
  "test_alloc",
  NULL
};

#endif /* #ifdef ERTS_ALC_INTERNAL__ */
# 817 "armv6-portbld-freebsd11.0/opt/smp/erl_alloc_types.h"


/* ------------------------------------------------------------------------- */
#endif /* #ifndef ERL_ALLOC_TYPES_H__ */
# 821 "armv6-portbld-freebsd11.0/opt/smp/erl_alloc_types.h"
# 25 "beam/erl_alloc.h" 2
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/erl_alloc.h"
# 1 "beam/erl_thr_progress.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread progress information. Used by lock free algorithms
 *              to determine when all involved threads are guaranteed to
 *              have passed a specific point of execution.
 *
 *              Usage instructions can be found in ert_thr_progress.c
 *
 * Author: 	Rickard Green
 */

#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
#define ERL_THR_PROGRESS_H__TSD_TYPE__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/erl_thr_progress.h"
# 35 "beam/erl_thr_progress.h"

#ifndef ERTS_SMP

#define erts_smp_thr_progress_block() ((void) 0)
#define erts_smp_thr_progress_unblock() ((void) 0)
#define erts_smp_thr_progress_is_blocking() 1

#else /* ERTS_SMP */
# 43 "beam/erl_thr_progress.h"

#define erts_smp_thr_progress_block erts_thr_progress_block
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking

void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);

typedef Uint64 ErtsThrPrgrVal;

#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */

typedef struct {
    ErtsThrPrgrVal next;
    ErtsThrPrgrVal current;
    int chk_next_ix;
    struct {
	int current;
	int waiting;
    } umrefc_ix;
} ErtsThrPrgrLeaderState;

typedef struct {
    int id;
    int is_managed;
    int is_blocking;
#ifdef ERTS_ENABLE_LOCK_CHECK
    int is_delaying; /* managed is always delaying */
#endif
# 73 "beam/erl_thr_progress.h"
    int is_temporary;

    /* --- Part below only for registered threads --- */

    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];

    /* --- Part below only for managed threads --- */

    int leader; /* Needs to be first in the managed threads part */
    int active;
    ErtsThrPrgrVal confirmed;
    ErtsThrPrgrLeaderState leader_state;
} ErtsThrPrgrData;

int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
void erts_thr_progress_fatal_error_wait(SWord timeout);

#endif /* ERTS_SMP */
# 91 "beam/erl_thr_progress.h"

typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
struct ErtsThrPrgrLaterOp_ {
#ifdef ERTS_SMP
    ErtsThrPrgrVal later;
#endif
# 97 "beam/erl_thr_progress.h"
    void (*func)(void *);
    void *data;
    ErtsThrPrgrLaterOp *next;
};

#endif
# 103 "beam/erl_thr_progress.h"

#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
#define ERL_THR_PROGRESS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 107 "beam/erl_thr_progress.h"
# 108 "beam/erl_thr_progress.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 108 "beam/erl_thr_progress.h"
# 109 "beam/erl_thr_progress.h"

#ifdef ERTS_SMP

/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))

extern erts_tsd_key_t erts_thr_prgr_data_key__;

#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t

typedef struct {
    void *arg;
    void (*wakeup)(void *);
    void (*prepare_wait)(void *);
    void (*wait)(void *);
    void (*finalize_wait)(void *);
} ErtsThrPrgrCallbacks;

typedef struct {
    ERTS_THR_PRGR_ATOMIC current;
} ErtsThrPrgr;

typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)

extern ErtsThrPrgr erts_thr_prgr__;

void erts_thr_progress_pre_init(void);
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
					       ErtsThrPrgrCallbacks *,
					       int);
void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
			      ErtsThrPrgrVal value);
int erts_thr_progress_update(ErtsSchedulerData *esdp);
int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
void erts_thr_progress_unmanaged_continue__(int umrefc_ix);

void erts_thr_progress_dbg_print_state(void);

ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);

ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
#endif
# 168 "beam/erl_thr_progress.h"
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
					    ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
}

ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_managed;
}

ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)
{
    if (erts_thr_progress_is_managed_thread())
	return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
    else
	return erts_thr_progress_unmanaged_delay__();
}

ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
{
    ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
	   || erts_thr_progress_is_managed_thread());
    if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
	erts_thr_progress_unmanaged_continue__(handle);
}

#ifdef ERTS_ENABLE_LOCK_CHECK

ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_delaying;
}

#endif
# 233 "beam/erl_thr_progress.h"

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
{
    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
	return ((ErtsThrPrgrVal) 0);
    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
	return ((ErtsThrPrgrVal) 1);
    else
	return val + ((ErtsThrPrgrVal) 2);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData *esdp)
{
    ErtsThrPrgrData *tpd;
    ErtsThrPrgrVal val;
    if (esdp) {
	tpd = &esdp->thr_progress_data;
    managed_thread:
	val = tpd->confirmed;
	ERTS_THR_MEMORY_BARRIER;
    }
    else {
	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
	if (tpd && tpd->is_managed)
	    goto managed_thread;
	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
    }
    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
    return erts_thr_progress_current_to_later__(val);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
    else
	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}

ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
{
    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
	/* May have wrapped... */
	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
	    /*
	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
	     * these ranges 'current' is considered later than 'val0'.
	     */
	    return 1;
	}
    }
    return val1 > val0;
}

ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
{
    if (this == val)
	return 1;
    return erts_thr_progress_has_passed__(this, val);
}

ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
}

ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    if (val1 == val2)
	return 0;
    if (erts_thr_progress_has_passed__(val1, val2))
	return 1;
    else
	return -1;
}	

ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)
{
    ErtsThrPrgrVal current = erts_thr_progress_current();
    return erts_thr_progress_has_reached_this(current, val);
}

#endif
# 326 "beam/erl_thr_progress.h"

#endif /* ERTS_SMP */
# 328 "beam/erl_thr_progress.h"

#endif
# 330 "beam/erl_thr_progress.h"
# 28 "beam/erl_alloc.h" 2
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc_util.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/erl_alloc.h"
# 1 "beam/erl_alloc_util.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2002-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_ALLOC_UTIL__
#define ERL_ALLOC_UTIL__

#define ERTS_ALCU_VSN_STR "3.0"

#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc_types.h"
#endif /* expanded by -frewrite-includes */
# 26 "beam/erl_alloc_util.h"
# 27 "beam/erl_alloc_util.h"

#define ERTS_AU_PREF_ALLOC_BITS 11
#define ERTS_AU_MAX_PREF_ALLOC_INSTANCES (1 << ERTS_AU_PREF_ALLOC_BITS)

typedef struct Allctr_t_ Allctr_t;

typedef struct {
    UWord ycs;
    UWord mmc;
    int   sac;
} AlcUInit_t;

typedef struct {
    char *name_prefix;
    ErtsAlcType_t alloc_no;
    int force;
    int ix;
    int ts;
    int tspec;
    int tpref;
    int ramv;
    int low_mem;   /* HALFWORD only */
    UWord sbct;
    UWord asbcst;
    UWord rsbcst;
    UWord rsbcmt;
    UWord rmbcmt;
    UWord mmbcs;
    UWord mmsbc;
    UWord mmmbc;
    UWord lmbcs;
    UWord smbcs;
    UWord mbcgs;
    int acul;

    void *fix;
    size_t *fix_type_size;
} AllctrInit_t;

typedef struct {
    UWord blocks;
    UWord carriers;
} AllctrSize_t;

typedef struct {
    UWord allocated;
    UWord used;
} ErtsAlcUFixInfo_t;

#ifndef SMALL_MEMORY

#define ERTS_DEFAULT_ALCU_INIT {                                           \
    1024*1024,		/* (bytes)  ycs:    sys_alloc carrier size       */\
    ~((UWord) 0),	/* (amount) mmc:    max mseg carriers            */\
    1			/* (bool)   sac:    sys_alloc carriers           */\
}

#define ERTS_DEFAULT_ALLCTR_INIT {                                         \
    NULL,                                                                  \
    ERTS_ALC_A_INVALID,	/* (number) alloc_no: allocator number           */\
    0,			/* (bool)   force:  force enabled                */\
    0,			/* (number) ix: instance index                   */\
    1,			/* (bool)   ts:     thread safe                  */\
    0,			/* (bool)   tspec:  thread specific              */\
    0,			/* (bool)   tpref:  thread preferred             */\
    0,			/* (bool)   ramv:   realloc always moves         */\
    0,			/* (bool)   low_mem: HALFWORD only               */\
    512*1024,		/* (bytes)  sbct:   sbc threshold                */\
    2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
    20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
    80,			/* (%)      rsbcmt: rel sbc move threshold       */\
    50,			/* (%)      rmbcmt: rel mbc move threshold       */\
    1024*1024,		/* (bytes)  mmbcs:  main multiblock carrier size */\
    256,		/* (amount) mmsbc:  max mseg sbcs                */\
    ~((UWord) 0),	/* (amount) mmmbc:  max mseg mbcs                */ \
    10*1024*1024,	/* (bytes)  lmbcs:  largest mbc size             */\
    1024*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
    10,			/* (amount) mbcgs:  mbc growth stages            */\
    0,			/* (%)      acul:  abandon carrier utilization limit */\
    /* --- Data not options -------------------------------------------- */\
    NULL,		/* (ptr)    fix                                  */\
    NULL		/* (ptr)    fix_type_size                        */\
}

#else /* if SMALL_MEMORY */
# 112 "beam/erl_alloc_util.h"

#define ERTS_DEFAULT_ALCU_INIT {                                           \
    128*1024,		/* (bytes)  ycs:    sys_alloc carrier size       */\
    1024,      		/* (amount) mmc:    max mseg carriers            */\
    1			/* (bool)   sac:    sys_alloc carriers           */\
}

#define ERTS_DEFAULT_ALLCTR_INIT {                                         \
    NULL,                                                                  \
    ERTS_ALC_A_INVALID,	/* (number) alloc_no: allocator number           */\
    0,			/* (bool)   force:  force enabled                */\
    0,			/* (number) ix: instance index                   */\
    1,			/* (bool)   ts:     thread safe                  */\
    0,			/* (bool)   tspec:  thread specific              */\
    0,			/* (bool)   tpref:  thread preferred             */\
    0,			/* (bool)   ramv:   realloc always moves         */\
    0,			/* (bool)   low_mem: HALFWORD only               */\
    64*1024,		/* (bytes)  sbct:   sbc threshold                */\
    2*1024*2024,	/* (amount) asbcst: abs sbc shrink threshold     */\
    20,			/* (%)      rsbcst: rel sbc shrink threshold     */\
    80,			/* (%)      rsbcmt: rel sbc move threshold       */\
    128*1024,		/* (bytes)  mmbcs:  main multiblock carrier size */\
    256,		/* (amount) mmsbc:  max mseg sbcs                */\
    ~((UWord) 0),	/* (amount) mmmbc:  max mseg mbcs                */ \
    1024*1024,		/* (bytes)  lmbcs:  largest mbc size             */\
    128*1024,		/* (bytes)  smbcs:  smallest mbc size            */\
    10,			/* (amount) mbcgs:  mbc growth stages            */\
    0,			/* (%)      acul:  abandon carrier utilization limit */\
    /* --- Data not options -------------------------------------------- */\
    NULL,		/* (ptr)    fix                                  */\
    NULL		/* (ptr)    fix_type_size                        */\
}

#endif
# 146 "beam/erl_alloc_util.h"

void *	erts_alcu_alloc(ErtsAlcType_t, void *, Uint);
void *	erts_alcu_realloc(ErtsAlcType_t, void *, void *, Uint);
void *	erts_alcu_realloc_mv(ErtsAlcType_t, void *, void *, Uint);
void	erts_alcu_free(ErtsAlcType_t, void *, void *);
#ifdef USE_THREADS
void *	erts_alcu_alloc_ts(ErtsAlcType_t, void *, Uint);
void *	erts_alcu_realloc_ts(ErtsAlcType_t, void *, void *, Uint);
void *	erts_alcu_realloc_mv_ts(ErtsAlcType_t, void *, void *, Uint);
void	erts_alcu_free_ts(ErtsAlcType_t, void *, void *);
#ifdef ERTS_SMP
void *	erts_alcu_alloc_thr_spec(ErtsAlcType_t, void *, Uint);
void *	erts_alcu_realloc_thr_spec(ErtsAlcType_t, void *, void *, Uint);
void *	erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t, void *, void *, Uint);
void	erts_alcu_free_thr_spec(ErtsAlcType_t, void *, void *);
void *	erts_alcu_alloc_thr_pref(ErtsAlcType_t, void *, Uint);
void *	erts_alcu_realloc_thr_pref(ErtsAlcType_t, void *, void *, Uint);
void *	erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint);
void	erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *);
#endif
# 166 "beam/erl_alloc_util.h"
#endif
# 167 "beam/erl_alloc_util.h"
Eterm	erts_alcu_au_info_options(int *, void *, Uint **, Uint *);
Eterm	erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *);
Eterm	erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *);
Eterm	erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *);
void	erts_alcu_init(AlcUInit_t *);
void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
			       ErtsAlcUFixInfo_t *, int);
#ifdef ERTS_SMP
void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
#endif
# 177 "beam/erl_alloc_util.h"
erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);

#endif
# 180 "beam/erl_alloc_util.h"

#if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
#define ERL_ALLOC_UTIL_IMPL__

#define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE		(((Uint32) 1) << 0)

#ifdef USE_THREADS
#define ERL_THREADS_EMU_INTERNAL__
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 188 "beam/erl_alloc_util.h"
# 189 "beam/erl_alloc_util.h"
#endif
# 190 "beam/erl_alloc_util.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 191 "beam/erl_alloc_util.h"
# 192 "beam/erl_alloc_util.h"

#undef ERTS_ALLOC_UTIL_HARD_DEBUG
#ifdef DEBUG
#  if 0
#    define ERTS_ALLOC_UTIL_HARD_DEBUG
#  endif
# 198 "beam/erl_alloc_util.h"
#endif
# 199 "beam/erl_alloc_util.h"

#undef MIN
#undef MAX
#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
#define FLOOR(X, I) (((X)/(I))*(I))
#define CEILING(X, I)  ((((X) - 1)/(I) + 1)*(I))

#undef  WORD_MASK
#define INV_WORD_MASK	((UWord) (sizeof(UWord) - 1))
#define WORD_MASK	(~INV_WORD_MASK)
#define WORD_FLOOR(X)	((X) & WORD_MASK)
#define WORD_CEILING(X)	WORD_FLOOR((X) + INV_WORD_MASK)

#undef  UNIT_MASK
#define INV_UNIT_MASK	((UWord) (sizeof(Unit_t) - 1))
#define UNIT_MASK	(~INV_UNIT_MASK)
#define UNIT_FLOOR(X)	((X) & UNIT_MASK)
#define UNIT_CEILING(X)	UNIT_FLOOR((X) + INV_UNIT_MASK)

#define FLG_MASK		INV_UNIT_MASK
#define SBC_BLK_SZ_MASK         UNIT_MASK
#define MBC_FBLK_SZ_MASK        UNIT_MASK
#define CARRIER_SZ_MASK         UNIT_MASK

#if ERTS_HAVE_MSEG_SUPER_ALIGNED \
    || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC)
#  ifndef MSEG_ALIGN_BITS
#    define ERTS_SUPER_ALIGN_BITS MSEG_ALIGN_BITS
#  else
# 229 "beam/erl_alloc_util.h"
#    define ERTS_SUPER_ALIGN_BITS 18
#  endif
# 231 "beam/erl_alloc_util.h"
#  ifdef ARCH_64 
#    define MBC_ABLK_OFFSET_BITS   24
#  else
# 234 "beam/erl_alloc_util.h"
#    define MBC_ABLK_OFFSET_BITS   9
     /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */
#  endif
# 237 "beam/erl_alloc_util.h"
#  define ERTS_SACRR_UNIT_SHIFT		ERTS_SUPER_ALIGN_BITS
#  define ERTS_SACRR_UNIT_SZ		(1 << ERTS_SACRR_UNIT_SHIFT)
#  define ERTS_SACRR_UNIT_MASK		((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT)
#  define ERTS_SACRR_UNIT_FLOOR(X)	((X) & ERTS_SACRR_UNIT_MASK)
#  define ERTS_SACRR_UNIT_CEILING(X)	ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK)
#  define ERTS_SA_MB_CARRIERS 1
#else
# 244 "beam/erl_alloc_util.h"
#  define ERTS_SA_MB_CARRIERS 0
#  define MBC_ABLK_OFFSET_BITS   0 /* no carrier offset in block header */
#endif
# 247 "beam/erl_alloc_util.h"
#if ERTS_HAVE_MSEG_SUPER_ALIGNED && !ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
#  define ERTS_SUPER_ALIGNED_MSEG_ONLY 1
#else
# 250 "beam/erl_alloc_util.h"
#  define ERTS_SUPER_ALIGNED_MSEG_ONLY 0
#endif
# 252 "beam/erl_alloc_util.h"

#if MBC_ABLK_OFFSET_BITS
#  define MBC_ABLK_OFFSET_SHIFT  (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS)
#  define MBC_ABLK_OFFSET_MASK   (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT)
#  define MBC_ABLK_SZ_MASK	(~MBC_ABLK_OFFSET_MASK & ~FLG_MASK)
#else
# 258 "beam/erl_alloc_util.h"
#  define MBC_ABLK_SZ_MASK	(~FLG_MASK)
#endif
# 260 "beam/erl_alloc_util.h"

#define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK)
#define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK)
#define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK)

#define CARRIER_SZ(C) \
  ((C)->chdr & CARRIER_SZ_MASK)

typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t;

#ifdef ERTS_SMP

typedef struct ErtsDoubleLink_t_ {
    struct ErtsDoubleLink_t_ *next;
    struct ErtsDoubleLink_t_ *prev;
}ErtsDoubleLink_t;

typedef struct {
    erts_atomic_t next;
    erts_atomic_t prev;
    Allctr_t *orig_allctr;      /* read-only while carrier is alive */
    ErtsThrPrgrVal thr_prgr;
    erts_atomic_t max_size;
    UWord abandon_limit;
    UWord blocks;
    UWord blocks_size;
    ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */
} ErtsAlcCPoolData_t;

#endif
# 290 "beam/erl_alloc_util.h"

typedef struct Carrier_t_ Carrier_t;
struct Carrier_t_ {
    UWord chdr;
    Carrier_t *next;
    Carrier_t *prev;
    erts_smp_atomic_t allctr;
#ifdef ERTS_SMP
    ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */
#endif
# 300 "beam/erl_alloc_util.h"
};

#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \
  ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK))

typedef struct {
    Carrier_t *first;
    Carrier_t *last;
} CarrierList_t;

typedef struct {
    UWord bhdr;
#if !MBC_ABLK_OFFSET_BITS
    Carrier_t *carrier;
#else
# 315 "beam/erl_alloc_util.h"
    union {
	Carrier_t *carrier;     /* if free */
	char       udata__[1];  /* if allocated */
    }u;
#endif
# 320 "beam/erl_alloc_util.h"
} Block_t;

#define THIS_FREE_BLK_HDR_FLG 	(((UWord) 1) << 0)
#define PREV_FREE_BLK_HDR_FLG 	(((UWord) 1) << 1)
#define LAST_BLK_HDR_FLG 	(((UWord) 1) << 2)

#define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\
    (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)

/*
 * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for
 * distinguishing empty mbc's from allocated blocks in
 * handle_delayed_dealloc().
 */
#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)

#define IS_FREE_LAST_MBC_BLK(B) \
    (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS)

#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG)
#define IS_MBC_BLK(B) (!IS_SBC_BLK((B)))
#define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \
			(B)->bhdr & THIS_FREE_BLK_HDR_FLG)

#if MBC_ABLK_OFFSET_BITS
#  define FBLK_TO_MBC(B) (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \
			  (B)->u.carrier)
#  define ABLK_TO_MBC(B) \
    (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \
     (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \
		  (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT))))
#  define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B))
#else
# 353 "beam/erl_alloc_util.h"
#  define FBLK_TO_MBC(B) ((B)->carrier)
#  define ABLK_TO_MBC(B) ((B)->carrier)
#  define BLK_TO_MBC(B)  ((B)->carrier)
#endif
# 357 "beam/erl_alloc_util.h"
#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B))

typedef UWord FreeBlkFtr_t; /* Footer of a free block */

typedef Uint64 CallCounter_t;

typedef struct {
    UWord		no;
    UWord		size;
} StatValues_t;

typedef struct {
    union {
	struct {
	    StatValues_t	mseg;
	    StatValues_t	sys_alloc;
	} norm;
    } curr;
    StatValues_t	max;
    StatValues_t	max_ever;
    struct {
	StatValues_t	curr;
	StatValues_t	max;
	StatValues_t	max_ever;
    } blocks;
} CarriersStats_t;

#ifdef ERTS_SMP

typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t;

union ErtsAllctrDDBlock_t_ {
    erts_atomic_t atmc_next;
    ErtsAllctrDDBlock_t *ptr_next;
};

typedef struct {
    ErtsAllctrDDBlock_t marker;
    erts_atomic_t last;
    erts_atomic_t um_refc[2];
    erts_atomic32_t um_refc_ix;
} ErtsDDTail_t;

typedef struct {
    /*
     * This structure needs to be cache line aligned for best
     * performance.
     */
    union {
	/* Modified by threads returning memory to this allocator */
	ErtsDDTail_t data;
	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsDDTail_t))];
    } tail;
    /*
     * Everything below this point is *only* accessed by the
     * thread owning the allocator.
     */
    struct {
	ErtsAllctrDDBlock_t *first;
	ErtsAllctrDDBlock_t *unref_end;
	struct {
	    ErtsThrPrgrVal thr_progress;
	    int thr_progress_reached;
	    int um_refc_ix;
	    ErtsAllctrDDBlock_t *unref_end;
	} next;
	int used_marker;
    } head;
} ErtsAllctrDDQueue_t;

#endif
# 428 "beam/erl_alloc_util.h"

typedef struct {
    size_t type_size;
    SWord list_size;
    void *list;
    union {
	struct {
	    SWord max_used;
	    SWord limit;
	    SWord allocated;
	    SWord used;
	} nocpool;
	struct {
	    int min_list_size;
	    int shrink_list;
	    UWord allocated;
	    UWord used;
	} cpool;
    } u;
} ErtsAlcFixList_t;

struct Allctr_t_ {
#ifdef ERTS_SMP
    struct {
	/*
	 * We want the queue at the beginning of
	 * the Allctr_t struct, due to cache line
	 * alignment reasons.
	 */
	ErtsAllctrDDQueue_t q;
	int		use;
	int		ix;
    } dd;
#endif
# 462 "beam/erl_alloc_util.h"

    /* Allocator name prefix */
    char *		name_prefix;

    /* Allocator number */
    ErtsAlcType_t	alloc_no;

    /* Instance index */
    int			ix;

    /* Alloc, realloc and free names as atoms */
    struct {
	Eterm		alloc;
	Eterm		realloc;
	Eterm		free;
    } name;

    /* Version string */
    char *		vsn_str;

    /* Options */
    int			t;
    int			ramv;
    Uint		sbc_threshold;
    Uint		sbc_move_threshold;
    Uint		mbc_move_threshold;
    Uint		main_carrier_size;
    Uint		max_mseg_sbcs;
    Uint		max_mseg_mbcs;
    Uint		largest_mbc_size;
    Uint		smallest_mbc_size;
    Uint		mbc_growth_stages;

#if HAVE_ERTS_MSEG
    ErtsMsegOpt_t	mseg_opt;
#endif
# 498 "beam/erl_alloc_util.h"

    /* */
    Uint		mbc_header_size;
    Uint		min_mbc_size;
    Uint		min_mbc_first_free_size;
    Uint		min_block_size;

    /* Carriers */
    CarrierList_t	mbc_list;
    CarrierList_t	sbc_list;
#ifdef ERTS_SMP
    struct {
	/* pooled_list, traitor list and dc_list contain only
           carriers _created_ by this allocator */
	ErtsDoubleLink_t pooled_list;
	ErtsDoubleLink_t traitor_list;
	CarrierList_t	 dc_list;

	UWord		abandon_limit;
	int		disable_abandon;
	int		check_limit_count;
	int		util_limit;
	struct {
	    erts_atomic_t	blocks_size;
	    erts_atomic_t	no_blocks;
	    erts_atomic_t	carriers_size;
	    erts_atomic_t	no_carriers;
	} stat;
    } cpool;
#endif
# 528 "beam/erl_alloc_util.h"

    /* Main carrier (if there is one) */
    Carrier_t *		main_carrier;

    /* Callback functions (first 4 are mandatory) */
    Block_t *		(*get_free_block)	(Allctr_t *, Uint,
						 Block_t *, Uint);
    void		(*link_free_block)	(Allctr_t *, Block_t *);
    void		(*unlink_free_block)	(Allctr_t *, Block_t *);
    Eterm		(*info_options)		(Allctr_t *, char *, int *,
						 void *, Uint **, Uint *);

    Uint		(*get_next_mbc_size)	(Allctr_t *);
    void		(*creating_mbc)		(Allctr_t *, Carrier_t *);
    void		(*destroying_mbc)	(Allctr_t *, Carrier_t *);

    /* The three callbacks below are needed to support carrier migration */
    void		(*add_mbc)		(Allctr_t *, Carrier_t *);
    void		(*remove_mbc)	        (Allctr_t *, Carrier_t *);
    UWord		(*largest_fblk_in_mbc)  (Allctr_t *, Carrier_t *);

    void		(*init_atoms)		(void);

#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG
    void		(*check_block)		(Allctr_t *, Block_t *,  int);
    void		(*check_mbc)		(Allctr_t *, Carrier_t *);
#endif
# 555 "beam/erl_alloc_util.h"

    int			fix_n_base;
    int			fix_shrink_scheduled;
    ErtsAlcFixList_t	*fix;

#ifdef USE_THREADS
    /* Mutex for this allocator */
    erts_mtx_t		mutex;
    int			thread_safe;
    struct {
	Allctr_t	*prev;
	Allctr_t	*next;
    } ts_list;

#endif
# 570 "beam/erl_alloc_util.h"

    int			atoms_initialized;

    int			stopped;

    /* Some statistics ... */
    struct {
	CallCounter_t	this_alloc;
	CallCounter_t	this_free;
	CallCounter_t	this_realloc;
	CallCounter_t	mseg_alloc;
	CallCounter_t	mseg_dealloc;
	CallCounter_t	mseg_realloc;
	CallCounter_t	sys_alloc;
	CallCounter_t	sys_free;
	CallCounter_t	sys_realloc;
    } calls;

    CarriersStats_t	sbcs;
    CarriersStats_t	mbcs;
    
#ifdef DEBUG
#ifdef USE_THREADS
    struct {
	int saved_tid;
	erts_tid_t tid;
    } debug;
#endif
# 598 "beam/erl_alloc_util.h"
#endif
# 599 "beam/erl_alloc_util.h"
};

int	erts_alcu_start(Allctr_t *, AllctrInit_t *);
void	erts_alcu_stop(Allctr_t *);

void	erts_alcu_verify_unused(Allctr_t *);
void	erts_alcu_verify_unused_ts(Allctr_t *allctr);

UWord	erts_alcu_test(UWord, UWord, UWord);

void erts_alcu_assert_failed(char* expr, char* file, int line, char *func);

#ifdef DEBUG
int is_sbc_blk(Block_t*);
#endif
# 614 "beam/erl_alloc_util.h"


#endif /* #if defined(GET_ERL_ALLOC_UTIL_IMPL)
	      && !defined(ERL_ALLOC_UTIL_IMPL__) */
# 618 "beam/erl_alloc_util.h"

# 30 "beam/erl_alloc.h" 2
#ifdef USE_THREADS
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_alloc.h"
# 1 "beam/erl_threads.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Description: Error checking thread interface to the ethread library.
 *              All functions terminates the emulator on failure.
 * Author: Rickard Green
 */

#ifndef ERL_THREAD_H__
#define ERL_THREAD_H__

/*
 * --- Documentation of atomics and memory barriers --------------------------
 *
 * The following explicit memory barriers exist:
 *   
 * - ERTS_THR_MEMORY_BARRIER
 *      Full memory barrier. Orders both loads, and stores. No
 *      load or store is allowed to be reordered over the
 *      barrier.
 * - ERTS_THR_WRITE_MEMORY_BARRIER
 *      Write barrier. Orders *only* stores. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_READ_MEMORY_BARRIER
 *      Read barrier. Orders *only* loads. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 *      Data dependency read barrier. Orders *only* loads
 *      according to data dependency across the barrier.
 *
 * If thread support has been disabled, these barriers will become no-ops.
 *
 * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
 * be enabled only in the SMP enabled runtime system.
 *
 * --- Atomic operations ---
 *
 * Atomics operations exist for 32-bit, word size, and double word size
 * integers. Function prototypes are listed below.
 *
 * Each function implementing an atomic operation exist with the following
 * implied memory barrier semantics. Not all combinations are useful, but
 * all of them exist for simplicity. <B> is suffix in function name:
 *
 * - <B>  - Description
 *
 * - mb   - Full memory barrier. Orders both loads, and
 *          stores before, and after the atomic operation.
 *          No load or store is allowed to be reordered
 *          over the atomic operation.
 * - relb - Release barrier. Orders both loads, and
 *          stores appearing *before* the atomic
 *          operation. These are not allowed to be
 *          reordered over the atomic operation.
 * - acqb - Acquire barrier. Orders both loads, and stores
 *          appearing *after* the atomic operation. These
 *          are not allowed to be reordered over the
 *          atomic operation.
 * - wb   - Write barrier. Orders *only* stores. These are
 *          not allowed to be reordered over the barrier.
 *          Store in atomic operation is ordered *after*
 *          the barrier.
 * - rb   - Read barrier. Orders *only* loads. These are
 *          not allowed to be reordered over the barrier.
 *          Load in atomic operation is ordered *before*
 *          the barrier. 
 * - ddrb - Data dependency read barrier. Orders *only*
 *          loads according to data dependency across the
 *          barrier. Load in atomic operation is ordered
 *          before the barrier.
 *
 * If thread support has been disabled, these functions are mapped to
 * functions that performs the same operation, but aren't atomic
 * and don't imply any memory barriers.
 *
 * If the atomic operations are prefixed with erts_smp_ instead of only
 * erts_ the atomic operations will only be atomic in the SMP enabled
 * runtime system, and will be mapped to non-atomic operations without
 * memory barriers in the runtime system without SMP support. Atomic
 * operations with erts_smp_ prefix should use the atomic types
 * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
 * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
 * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
 * are the same.
 *
 * --- 32-bit atomic operations ---
 *
 * The following 32-bit atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above). Note
 * that sizeof(erts_atomic32_t) might be larger than 4!
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
 *                               erts_aint32_t val);
 *
 * Set value:
 *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Read; returns current value:
 *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment:
 *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
 *
 * Decrement:
 *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Add value:
 *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
 *                                             erts_aint32_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
 *                                        erts_aint32_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
 *                                           erts_aint32_t new,
 *                                           erts_aint32_t exp);
 *
 * --- Word size atomic operations ---
 *
 * The following word size (same size as sizeof(void *)) atomic
 * operations exist. <B> should be replaced with a supported
 * memory barrier (see above). Note that sizeof(erts_atomic_t)
 * might be larger than sizeof(void *)!
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
 *                             erts_aint_t val);
 *
 * Set value;
 *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Read; returns current value:
 *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
 *
 * Increment:
 *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
 *
 * Decrement:
 *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Add value:
 *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
 *                                         erts_aint_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
 *                                    erts_aint_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
 *                                       erts_aint_t new,
 *                                       erts_aint_t exp);
 *
 * --- Double word size atomic operations ---
 *
 * The following double word atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above).
 *
 * Note that sizeof(erts_dw_atomic_t) usually is larger than
 * 2*sizeof(void *)!
 * 
 * The erts_dw_aint_t data type should be accessed as if it was defined
 * like this:
 *
 *     typedef struct {
 *         erts_aint_t sint[2];
 *     } erts_dw_aint_t;
 *
 *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
 *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
 *                           erts_dw_aint_t *val);
 *
 * Set; value is written into 'val':
 *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
 *                               erts_dw_aint_t *val);
 *
 * Read; value is written into 'val':
 *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
 *                                erts_dw_aint_t *val);
 *
 * Compare and exchange; returns a value != 0 if exchange was
 * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
 * contains the same value as in memory when the function is called,
 * 'new' is written to memory; otherwise, not. If exchange was not
 * made, 'exp_act' contains the actual value in memory:
 *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
 *                                  erts_dw_aint_t *new,
 *                                  erts_dw_aint_t *exp_act);
 */

#define ERTS_SPIN_BODY ETHR_SPIN_BODY

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 260 "beam/erl_threads.h"
# 261 "beam/erl_threads.h"

#ifdef USE_THREADS

#define ETHR_TRY_INLINE_FUNCS
#if 0 /* expanded by -frewrite-includes */
#include "ethread.h"
#endif /* expanded by -frewrite-includes */
# 265 "beam/erl_threads.h"
# 266 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 266 "beam/erl_threads.h"
# 267 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_count.h"
#endif /* expanded by -frewrite-includes */
# 267 "beam/erl_threads.h"
# 268 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 268 "beam/erl_threads.h"
# 269 "beam/erl_threads.h"

#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
/*
 * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
 * only seen this bug in glibc versions before 2.4. Note that condition
 * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
 * this bug since these implementations may use mutexes internally.
 */
#  define ERTS_THR_HAVE_BUSY_DESTROY_BUG
#endif
# 279 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__)
#define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__)
#endif
# 296 "beam/erl_threads.h"

#define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
typedef ethr_thr_opts erts_thr_opts_t;
typedef ethr_init_data erts_thr_init_data_t;
typedef ethr_late_init_data erts_thr_late_init_data_t;
typedef ethr_tid erts_tid_t;

/* mutex */
typedef struct {
    ethr_mutex mtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 309 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 312 "beam/erl_threads.h"

} erts_mtx_t;
typedef ethr_cond erts_cnd_t;

/* rwmutex */
typedef struct {
    ethr_rwmutex rwmtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 322 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 325 "beam/erl_threads.h"
} erts_rwmtx_t;

#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER
#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
typedef ethr_rwmutex_opt erts_rwmtx_opt_t;

typedef ethr_tsd_key erts_tsd_key_t;
typedef ethr_ts_event erts_tse_t;
#define erts_dw_aint_t ethr_dw_sint_t
#define erts_dw_atomic_t ethr_dw_atomic_t
#define erts_aint_t ethr_sint_t
#define erts_atomic_t ethr_atomic_t
#define erts_aint32_t ethr_sint32_t 
#define erts_atomic32_t ethr_atomic32_t

#if defined(ARCH_32)
#  define erts_atomic64_t ethr_dw_atomic_t
#  define erts_aint64_t ethr_sint64_t
#elif defined(ARCH_64)
# 352 "beam/erl_threads.h"
#  define erts_atomic64_t ethr_atomic_t
#  define erts_aint64_t ethr_sint_t
#else
# 355 "beam/erl_threads.h"
#  error "Not supported architecture"
#endif
# 357 "beam/erl_threads.h"

#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD

/* spinlock */
typedef struct {
    ethr_spinlock_t slck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 367 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 370 "beam/erl_threads.h"
} erts_spinlock_t;

/* rwlock */
typedef struct {
    ethr_rwlock_t rwlck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 378 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 381 "beam/erl_threads.h"
} erts_rwlock_t;

__decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
                                 /* implemented in erl_init.c */

#define ERTS_THR_INIT_DATA_DEF_INITER	ETHR_INIT_DATA_DEFAULT_INITER
#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
                                        ETHR_LATE_INIT_DATA_DEFAULT_INITER

#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
#  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
#endif
# 393 "beam/erl_threads.h"

#else /* #ifdef USE_THREADS */
# 395 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#define ERTS_THR_OPTS_DEFAULT_INITER 0
typedef int erts_thr_opts_t;
typedef int erts_thr_init_data_t;
typedef int erts_thr_late_init_data_t;
typedef int erts_tid_t;
typedef int erts_mtx_t;
typedef int erts_cnd_t;
#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_RWMTX_TYPE_NORMAL 0
#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_RWMTX_LONG_LIVED 0
#define ERTS_RWMTX_SHORT_LIVED 0
#define ERTS_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_rwmtx_opt_t;
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;

typedef struct { SWord sint[2]; } erts_dw_aint_t;
typedef SWord erts_aint_t;
typedef Sint32 erts_aint32_t;
typedef Sint64 erts_aint64_t;

#define erts_dw_atomic_t erts_dw_aint_t
#define erts_atomic_t erts_aint_t
#define erts_atomic32_t erts_aint32_t
#define erts_atomic64_t erts_aint64_t

#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
#else
# 439 "beam/erl_threads.h"
typedef struct { int gcc_is_buggy; } erts_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif
# 442 "beam/erl_threads.h"

#ifdef WORDS_BIGENDIAN
#define ERTS_DW_AINT_LOW_WORD 1
#define ERTS_DW_AINT_HIGH_WORD 0
#else
# 447 "beam/erl_threads.h"
#define ERTS_DW_AINT_LOW_WORD 0
#define ERTS_DW_AINT_HIGH_WORD 1
#endif
# 450 "beam/erl_threads.h"

#define ERTS_MTX_INITER			0
#define ERTS_CND_INITER			0
#define ERTS_THR_INIT_DATA_DEF_INITER	0

#define ERTS_HAVE_REC_MTX_INIT		1

#endif /* #ifdef USE_THREADS */
# 458 "beam/erl_threads.h"

#define erts_no_dw_atomic_t erts_dw_aint_t
#define erts_no_atomic_t erts_aint_t
#define erts_no_atomic32_t erts_aint32_t
#define erts_no_atomic64_t erts_aint64_t

#define ERTS_AINT_NULL ((erts_aint_t) NULL)

#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))

ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
				     void *arg, erts_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid);
ERTS_GLB_INLINE void erts_thr_exit(void *res);
ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
				     int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
					 Uint16 opt, int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
					    char *name,
					    Eterm extra,
					    int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file,
				       unsigned int line);
ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file,
				     unsigned int line);
#else
# 499 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx);
#endif
# 502 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
					   erts_rwmtx_opt_t *opt,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
				       char *name,
				       Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
					 erts_rwmtx_opt_t *opt,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 529 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
#endif
# 534 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);

ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *val,
					      erts_no_dw_atomic_t *old_val);
ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
						    erts_aint_t i);
ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
						    erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
						     erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
						erts_aint_t new);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
						   erts_aint_t new,
						   erts_aint_t expected);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var,
						     erts_aint_t mask,
						     erts_aint_t set);
ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
							erts_aint32_t i);
ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
							erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
							 erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
						    erts_aint32_t new);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
						       erts_aint32_t new,
						       erts_aint32_t expected);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
							 erts_aint32_t mask,
							 erts_aint32_t set);
ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
							erts_aint64_t i);
ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
							erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
							 erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
						    erts_aint64_t new);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
						       erts_aint64_t new,
						       erts_aint64_t expected);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
							 erts_aint64_t mask,
							 erts_aint64_t set);

ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
					      char *name,
					      Eterm extra,
					      Uint16 opt);
ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
					  char *name,
					  Eterm extra);
ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
					char *name);
ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line);
#else
# 626 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
#endif
# 628 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_rwlock_init_x(erts_rwlock_t *lock,
					char *name,
					Eterm extra);
ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
				      char *name);
ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
#else
# 640 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_read_lock(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
#endif
# 643 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname);
ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
ERTS_GLB_INLINE int erts_thr_get_main_status(void);
ERTS_GLB_INLINE void erts_thr_yield(void);


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS
#define ERTS_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
				      sigset_t *oset);
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);

#ifdef USE_THREADS
ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
#endif
# 674 "beam/erl_threads.h"

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 676 "beam/erl_threads.h"

#ifdef USE_THREADS

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \
do {									\
    Type act = ReadOp((VarP));						\
    while (1) {								\
	Type exp = act;							\
	Type new = exp & ~(Mask);					\
	new |= ((Mask) & (Set));					\
	act = CmpxchgOp((VarP), new, exp);				\
	if (act == exp)							\
	    return act;							\
    }									\
} while (0)
#endif
# 750 "beam/erl_threads.h"

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var);
ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var);

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of this file for info on atomics.
 */

/* Double word size atomics */

#define erts_dw_atomic_init_nob ethr_dw_atomic_init
#define erts_dw_atomic_set_nob ethr_dw_atomic_set
#define erts_dw_atomic_read_nob ethr_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb

#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb

#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb

#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb

#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb

#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    sint[0] = val->sint[0];
    sint[1] = val->sint[1];    
}

ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    val->sint[0] = sint[0];
    val->sint[1] = sint[1];
}

#endif
# 825 "beam/erl_threads.h"

/* Word size atomics */

#define erts_atomic_init_nob ethr_atomic_init
#define erts_atomic_set_nob ethr_atomic_set
#define erts_atomic_read_nob ethr_atomic_read
#define erts_atomic_inc_read_nob ethr_atomic_inc_read
#define erts_atomic_dec_read_nob ethr_atomic_dec_read
#define erts_atomic_inc_nob ethr_atomic_inc
#define erts_atomic_dec_nob ethr_atomic_dec
#define erts_atomic_add_read_nob ethr_atomic_add_read
#define erts_atomic_add_nob ethr_atomic_add
#define erts_atomic_read_bor_nob ethr_atomic_read_bor
#define erts_atomic_read_band_nob ethr_atomic_read_band
#define erts_atomic_xchg_nob ethr_atomic_xchg
#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg,
			    var, mask, set);
}
#endif
# 854 "beam/erl_threads.h"

#define erts_atomic_init_mb ethr_atomic_init_mb
#define erts_atomic_set_mb ethr_atomic_set_mb
#define erts_atomic_read_mb ethr_atomic_read_mb
#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic_inc_mb ethr_atomic_inc_mb
#define erts_atomic_dec_mb ethr_atomic_dec_mb
#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic_add_mb ethr_atomic_add_mb
#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 881 "beam/erl_threads.h"

#define erts_atomic_init_acqb ethr_atomic_init_acqb
#define erts_atomic_set_acqb ethr_atomic_set_acqb
#define erts_atomic_read_acqb ethr_atomic_read_acqb
#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic_add_acqb ethr_atomic_add_acqb
#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 908 "beam/erl_threads.h"

#define erts_atomic_init_relb ethr_atomic_init_relb
#define erts_atomic_set_relb ethr_atomic_set_relb
#define erts_atomic_read_relb ethr_atomic_read_relb
#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic_inc_relb ethr_atomic_inc_relb
#define erts_atomic_dec_relb ethr_atomic_dec_relb
#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic_add_relb ethr_atomic_add_relb
#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 935 "beam/erl_threads.h"

#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 962 "beam/erl_threads.h"

#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic_inc_rb ethr_atomic_inc_rb
#define erts_atomic_dec_rb ethr_atomic_dec_rb
#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic_add_rb ethr_atomic_add_rb
#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 989 "beam/erl_threads.h"

#define erts_atomic_init_wb ethr_atomic_init_wb
#define erts_atomic_set_wb ethr_atomic_set_wb
#define erts_atomic_read_wb ethr_atomic_read_wb
#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic_inc_wb ethr_atomic_inc_wb
#define erts_atomic_dec_wb ethr_atomic_dec_wb
#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic_add_wb ethr_atomic_add_wb
#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1018 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    return *sint;
}

#endif
# 1036 "beam/erl_threads.h"

/* 32-bit atomics */

#define erts_atomic32_init_nob ethr_atomic32_init
#define erts_atomic32_set_nob ethr_atomic32_set
#define erts_atomic32_read_nob ethr_atomic32_read
#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
#define erts_atomic32_inc_nob ethr_atomic32_inc
#define erts_atomic32_dec_nob ethr_atomic32_dec
#define erts_atomic32_add_read_nob ethr_atomic32_add_read
#define erts_atomic32_add_nob ethr_atomic32_add
#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
#define erts_atomic32_read_band_nob ethr_atomic32_read_band
#define erts_atomic32_xchg_nob ethr_atomic32_xchg
#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg,
			    var, mask, set);
}
#endif
# 1065 "beam/erl_threads.h"

#define erts_atomic32_init_mb ethr_atomic32_init_mb
#define erts_atomic32_set_mb ethr_atomic32_set_mb
#define erts_atomic32_read_mb ethr_atomic32_read_mb
#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
#define erts_atomic32_add_mb ethr_atomic32_add_mb
#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 1092 "beam/erl_threads.h"

#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 1119 "beam/erl_threads.h"

#define erts_atomic32_init_relb ethr_atomic32_init_relb
#define erts_atomic32_set_relb ethr_atomic32_set_relb
#define erts_atomic32_read_relb ethr_atomic32_read_relb
#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
#define erts_atomic32_add_relb ethr_atomic32_add_relb
#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 1146 "beam/erl_threads.h"

#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 1173 "beam/erl_threads.h"

#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
#define erts_atomic32_add_rb ethr_atomic32_add_rb
#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 1200 "beam/erl_threads.h"

#define erts_atomic32_init_wb ethr_atomic32_init_wb
#define erts_atomic32_set_wb ethr_atomic32_set_wb
#define erts_atomic32_read_wb ethr_atomic32_read_wb
#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
#define erts_atomic32_add_wb ethr_atomic32_add_wb
#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1229 "beam/erl_threads.h"

#undef ERTS_ATOMIC_BSET_IMPL__

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    return *sint;
}

#endif
# 1249 "beam/erl_threads.h"

/* 64-bit atomics */

#if defined(ARCH_64)

#define erts_atomic64_init_nob ethr_atomic_init
#define erts_atomic64_set_nob ethr_atomic_set
#define erts_atomic64_read_nob ethr_atomic_read
#define erts_atomic64_inc_read_nob ethr_atomic_inc_read
#define erts_atomic64_dec_read_nob ethr_atomic_dec_read
#define erts_atomic64_inc_nob ethr_atomic_inc
#define erts_atomic64_dec_nob ethr_atomic_dec
#define erts_atomic64_add_read_nob ethr_atomic_add_read
#define erts_atomic64_add_nob ethr_atomic_add
#define erts_atomic64_read_bor_nob ethr_atomic_read_bor
#define erts_atomic64_read_band_nob ethr_atomic_read_band
#define erts_atomic64_xchg_nob ethr_atomic_xchg
#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob

#define erts_atomic64_init_mb ethr_atomic_init_mb
#define erts_atomic64_set_mb ethr_atomic_set_mb
#define erts_atomic64_read_mb ethr_atomic_read_mb
#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic64_inc_mb ethr_atomic_inc_mb
#define erts_atomic64_dec_mb ethr_atomic_dec_mb
#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic64_add_mb ethr_atomic_add_mb
#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb

#define erts_atomic64_init_acqb ethr_atomic_init_acqb
#define erts_atomic64_set_acqb ethr_atomic_set_acqb
#define erts_atomic64_read_acqb ethr_atomic_read_acqb
#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic64_add_acqb ethr_atomic_add_acqb
#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_atomic64_init_relb ethr_atomic_init_relb
#define erts_atomic64_set_relb ethr_atomic_set_relb
#define erts_atomic64_read_relb ethr_atomic_read_relb
#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic64_inc_relb ethr_atomic_inc_relb
#define erts_atomic64_dec_relb ethr_atomic_dec_relb
#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic64_add_relb ethr_atomic_add_relb
#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb

#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_atomic64_init_rb ethr_atomic_init_rb
#define erts_atomic64_set_rb ethr_atomic_set_rb
#define erts_atomic64_read_rb ethr_atomic_read_rb
#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic64_inc_rb ethr_atomic_inc_rb
#define erts_atomic64_dec_rb ethr_atomic_dec_rb
#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic64_add_rb ethr_atomic_add_rb
#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb

#define erts_atomic64_init_wb ethr_atomic_init_wb
#define erts_atomic64_set_wb ethr_atomic_set_wb
#define erts_atomic64_read_wb ethr_atomic_read_wb
#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic64_inc_wb ethr_atomic_inc_wb
#define erts_atomic64_dec_wb ethr_atomic_dec_wb
#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic64_add_wb ethr_atomic_add_wb
#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb

#define erts_atomic64_set_dirty erts_atomic_set_dirty
#define erts_atomic64_read_dirty erts_atomic_read_dirty

#elif defined(ARCH_32)
# 1363 "beam/erl_threads.h"

#undef ERTS_ATOMIC64_OPS_DECL__

#define ERTS_ATOMIC64_OPS_DECL__(BARRIER)				\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)

ERTS_ATOMIC64_OPS_DECL__(nob);
ERTS_ATOMIC64_OPS_DECL__(mb);
ERTS_ATOMIC64_OPS_DECL__(acqb);
ERTS_ATOMIC64_OPS_DECL__(relb);
ERTS_ATOMIC64_OPS_DECL__(ddrb);
ERTS_ATOMIC64_OPS_DECL__(rb);
ERTS_ATOMIC64_OPS_DECL__(wb);

#undef ERTS_ATOMIC64_OPS_DECL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

/*
 * The ethr_dw_atomic_*_nob() functions below
 * are here to make it possible for the
 * ERTS_ATOMIC64_OPS_IMPL__() to map erts
 * barriers to ethread barriers...
 */
static ERTS_INLINE void
ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_init(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
		       ethr_dw_sint_t *val)
{
    ethr_dw_atomic_set(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_read(var, val);
}

static ERTS_INLINE int
ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
			   ethr_dw_sint_t *new,
			   ethr_dw_sint_t *xchg)
{
    return ethr_dw_atomic_cmpxchg(var, new, xchg);
}

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#ifdef ETHR_SU_DW_NAINT_T__
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((erts_aint64_t) DW.dw_sint)
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
#else /* !ETHR_SU_DW_NAINT_T__ */
# 1470 "beam/erl_threads.h"
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32)		\
     | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD])		\
	& ((erts_aint64_t) 0xffffffff)))
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    do {								\
	DW.sint[ETHR_DW_SINT_LOW_WORD] =				\
	    (ethr_sint_t) (AINT64 & 0xffffffff);			\
	DW.sint[ETHR_DW_SINT_HIGH_WORD] = 				\
	    (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff);		\
    } while (0)
#endif /* !ETHR_SU_DW_NAINT_T__ */
# 1482 "beam/erl_threads.h"

#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, 			\
					AVarP, XchgVar, NewVar, 	\
					ModificationCode)		\
do {									\
    ethr_dw_sint_t dw_xchg__, dw_new__;					\
    ethr_dw_atomic_read(AVarP, &dw_xchg__);				\
    do {								\
	XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__);			\
	{								\
	    ModificationCode;						\
	}								\
	ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar);			\
    } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__));		\
} while (0)

#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER)				\
    									\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_init_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_set_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    ethr_dw_sint_t dw;							\
    ethr_dw_atomic_read_ ## BARRIER(var, &dw);				\
    return ERTS_DW_SINT_TO_AINT64__(dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg | val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg & val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = val);				\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp)			\
{									\
    ethr_dw_sint_t dw_xchg, dw_new;					\
    ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp);				\
    ERTS_AINT64_TO_DW_SINT__(dw_new, new);				\
    if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg))	\
	return exp;							\
    return ERTS_DW_SINT_TO_AINT64__(dw_xchg);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    {					\
					new = xchg & ~mask;		\
					new |= mask & set;		\
				    });					\
    return xchg;							\
}

ERTS_ATOMIC64_OPS_IMPL__(nob)
ERTS_ATOMIC64_OPS_IMPL__(mb)
ERTS_ATOMIC64_OPS_IMPL__(acqb)
ERTS_ATOMIC64_OPS_IMPL__(relb)
ERTS_ATOMIC64_OPS_IMPL__(ddrb)
ERTS_ATOMIC64_OPS_IMPL__(rb)
ERTS_ATOMIC64_OPS_IMPL__(wb)

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    ethr_dw_sint_t dw;
    ERTS_AINT64_TO_DW_SINT__(dw, val);
    sint[0] = dw.sint[0];
    sint[1] = dw.sint[1];
}

ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var)
{
    ethr_sint_t *sint;
    ethr_dw_sint_t dw;
    sint = ethr_dw_atomic_addr(var);
    dw.sint[0] = sint[0];
    dw.sint[1] = sint[1];
    return ERTS_DW_SINT_TO_AINT64__(dw);
}

#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1682 "beam/erl_threads.h"

#endif /* ARCH_32 */
# 1684 "beam/erl_threads.h"

#else /* !USE_THREADS */
# 1686 "beam/erl_threads.h"

/* Double word size atomics */

#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_atomic_init_nob erts_no_atomic_set
#define erts_atomic_set_nob erts_no_atomic_set
#define erts_atomic_read_nob erts_no_atomic_read
#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_atomic_inc_nob erts_no_atomic_inc
#define erts_atomic_dec_nob erts_no_atomic_dec
#define erts_atomic_add_read_nob erts_no_atomic_add_read
#define erts_atomic_add_nob erts_no_atomic_add
#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_atomic_read_band_nob erts_no_atomic_read_band
#define erts_atomic_xchg_nob erts_no_atomic_xchg
#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_atomic_init_mb erts_no_atomic_set
#define erts_atomic_set_mb erts_no_atomic_set
#define erts_atomic_read_mb erts_no_atomic_read
#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_atomic_inc_mb erts_no_atomic_inc
#define erts_atomic_dec_mb erts_no_atomic_dec
#define erts_atomic_add_read_mb erts_no_atomic_add_read
#define erts_atomic_add_mb erts_no_atomic_add
#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_atomic_read_band_mb erts_no_atomic_read_band
#define erts_atomic_xchg_mb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_atomic_init_acqb erts_no_atomic_set
#define erts_atomic_set_acqb erts_no_atomic_set
#define erts_atomic_read_acqb erts_no_atomic_read
#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_atomic_inc_acqb erts_no_atomic_inc
#define erts_atomic_dec_acqb erts_no_atomic_dec
#define erts_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_atomic_add_acqb erts_no_atomic_add
#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_atomic_init_relb erts_no_atomic_set
#define erts_atomic_set_relb erts_no_atomic_set
#define erts_atomic_read_relb erts_no_atomic_read
#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_atomic_inc_relb erts_no_atomic_inc
#define erts_atomic_dec_relb erts_no_atomic_dec
#define erts_atomic_add_read_relb erts_no_atomic_add_read
#define erts_atomic_add_relb erts_no_atomic_add
#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_atomic_read_band_relb erts_no_atomic_read_band
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_atomic_init_ddrb erts_no_atomic_set
#define erts_atomic_set_ddrb erts_no_atomic_set
#define erts_atomic_read_ddrb erts_no_atomic_read
#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_atomic_inc_ddrb erts_no_atomic_inc
#define erts_atomic_dec_ddrb erts_no_atomic_dec
#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_atomic_add_ddrb erts_no_atomic_add
#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
#define erts_atomic_read_rb erts_no_atomic_read
#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_atomic_inc_rb erts_no_atomic_inc
#define erts_atomic_dec_rb erts_no_atomic_dec
#define erts_atomic_add_read_rb erts_no_atomic_add_read
#define erts_atomic_add_rb erts_no_atomic_add
#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_atomic_read_band_rb erts_no_atomic_read_band
#define erts_atomic_xchg_rb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_atomic_init_wb erts_no_atomic_set
#define erts_atomic_set_wb erts_no_atomic_set
#define erts_atomic_read_wb erts_no_atomic_read
#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_atomic_inc_wb erts_no_atomic_inc
#define erts_atomic_dec_wb erts_no_atomic_dec
#define erts_atomic_add_read_wb erts_no_atomic_add_read
#define erts_atomic_add_wb erts_no_atomic_add
#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_atomic_read_band_wb erts_no_atomic_read_band
#define erts_atomic_xchg_wb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_atomic_set_dirty erts_no_atomic_set
#define erts_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_atomic32_init_nob erts_no_atomic32_set
#define erts_atomic32_set_nob erts_no_atomic32_set
#define erts_atomic32_read_nob erts_no_atomic32_read
#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_atomic32_inc_nob erts_no_atomic32_inc
#define erts_atomic32_dec_nob erts_no_atomic32_dec
#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_atomic32_add_nob erts_no_atomic32_add
#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_atomic32_init_mb erts_no_atomic32_set
#define erts_atomic32_set_mb erts_no_atomic32_set
#define erts_atomic32_read_mb erts_no_atomic32_read
#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_atomic32_inc_mb erts_no_atomic32_inc
#define erts_atomic32_dec_mb erts_no_atomic32_dec
#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_atomic32_add_mb erts_no_atomic32_add
#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_atomic32_init_acqb erts_no_atomic32_set
#define erts_atomic32_set_acqb erts_no_atomic32_set
#define erts_atomic32_read_acqb erts_no_atomic32_read
#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_atomic32_add_acqb erts_no_atomic32_add
#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_atomic32_init_relb erts_no_atomic32_set
#define erts_atomic32_set_relb erts_no_atomic32_set
#define erts_atomic32_read_relb erts_no_atomic32_read
#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_atomic32_inc_relb erts_no_atomic32_inc
#define erts_atomic32_dec_relb erts_no_atomic32_dec
#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_atomic32_add_relb erts_no_atomic32_add
#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_atomic32_init_ddrb erts_no_atomic32_set
#define erts_atomic32_set_ddrb erts_no_atomic32_set
#define erts_atomic32_read_ddrb erts_no_atomic32_read
#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_atomic32_add_ddrb erts_no_atomic32_add
#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
#define erts_atomic32_read_rb erts_no_atomic32_read
#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_atomic32_inc_rb erts_no_atomic32_inc
#define erts_atomic32_dec_rb erts_no_atomic32_dec
#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_atomic32_add_rb erts_no_atomic32_add
#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_atomic32_init_wb erts_no_atomic32_set
#define erts_atomic32_set_wb erts_no_atomic32_set
#define erts_atomic32_read_wb erts_no_atomic32_read
#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_atomic32_inc_wb erts_no_atomic32_inc
#define erts_atomic32_dec_wb erts_no_atomic32_dec
#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_atomic32_add_wb erts_no_atomic32_add
#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_atomic32_set_dirty erts_no_atomic32_set
#define erts_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_atomic64_init_nob erts_no_atomic64_set
#define erts_atomic64_set_nob erts_no_atomic64_set
#define erts_atomic64_read_nob erts_no_atomic64_read
#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_atomic64_inc_nob erts_no_atomic64_inc
#define erts_atomic64_dec_nob erts_no_atomic64_dec
#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_atomic64_add_nob erts_no_atomic64_add
#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_atomic64_init_mb erts_no_atomic64_set
#define erts_atomic64_set_mb erts_no_atomic64_set
#define erts_atomic64_read_mb erts_no_atomic64_read
#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_atomic64_inc_mb erts_no_atomic64_inc
#define erts_atomic64_dec_mb erts_no_atomic64_dec
#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_atomic64_add_mb erts_no_atomic64_add
#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_atomic64_init_acqb erts_no_atomic64_set
#define erts_atomic64_set_acqb erts_no_atomic64_set
#define erts_atomic64_read_acqb erts_no_atomic64_read
#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_atomic64_add_acqb erts_no_atomic64_add
#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_atomic64_init_relb erts_no_atomic64_set
#define erts_atomic64_set_relb erts_no_atomic64_set
#define erts_atomic64_read_relb erts_no_atomic64_read
#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_atomic64_inc_relb erts_no_atomic64_inc
#define erts_atomic64_dec_relb erts_no_atomic64_dec
#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_atomic64_add_relb erts_no_atomic64_add
#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_atomic64_init_ddrb erts_no_atomic64_set
#define erts_atomic64_set_ddrb erts_no_atomic64_set
#define erts_atomic64_read_ddrb erts_no_atomic64_read
#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_atomic64_add_ddrb erts_no_atomic64_add
#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_atomic64_init_rb erts_no_atomic64_set
#define erts_atomic64_set_rb erts_no_atomic64_set
#define erts_atomic64_read_rb erts_no_atomic64_read
#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_atomic64_inc_rb erts_no_atomic64_inc
#define erts_atomic64_dec_rb erts_no_atomic64_dec
#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_atomic64_add_rb erts_no_atomic64_add
#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_atomic64_init_wb erts_no_atomic64_set
#define erts_atomic64_set_wb erts_no_atomic64_set
#define erts_atomic64_read_wb erts_no_atomic64_read
#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_atomic64_inc_wb erts_no_atomic64_inc
#define erts_atomic64_dec_wb erts_no_atomic64_dec
#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_atomic64_add_wb erts_no_atomic64_add
#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_atomic64_set_dirty erts_no_atomic64_set
#define erts_atomic64_read_dirty erts_no_atomic64_read

#endif /* !USE_THREADS */
# 2058 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_thr_init(erts_thr_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_init(id);
    if (res)
	erts_thr_fatal_error(res, "initialize thread library");
#endif
# 2069 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_late_init(erts_thr_late_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_late_init(id);
    if (res)
	erts_thr_fatal_error(res, "complete initialization of thread library");
#endif
# 2079 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
		erts_thr_opts_t *opts)
{
#ifdef USE_THREADS
    int res = ethr_thr_create(tid, func, arg, opts);
    if (res)
	erts_thr_fatal_error(res, "create thread");
#endif
# 2090 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_join(erts_tid_t tid, void **thr_res)
{
#ifdef USE_THREADS
    int res = ethr_thr_join(tid, thr_res);
    if (res)
	erts_thr_fatal_error(res, "join thread");
#endif
# 2100 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_detach(erts_tid_t tid)
{
#ifdef USE_THREADS
    int res = ethr_thr_detach(tid);
    if (res)
	erts_thr_fatal_error(res, "detach thread");
#endif
# 2111 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_exit(void *res)
{
#ifdef USE_THREADS
    ethr_thr_exit(res);
    erts_thr_fatal_error(0, "terminate thread");
#endif
# 2121 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_install_exit_handler(void (*exit_handler)(void))
{
#ifdef USE_THREADS
    int res = ethr_install_exit_handler(exit_handler);
    if (res != 0)
	erts_thr_fatal_error(res, "install thread exit handler");
#endif
# 2131 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tid_t
erts_thr_self(void)
{
#ifdef USE_THREADS
    return ethr_self();
#else
# 2139 "beam/erl_threads.h"
    return 0;
#endif
# 2141 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
{
#ifdef USE_THREADS
    return ethr_getname(tid, buf, len);
#else
# 2149 "beam/erl_threads.h"
    return -1;
#endif
# 2151 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
erts_equal_tids(erts_tid_t x, erts_tid_t y)
{
#ifdef USE_THREADS
    return ethr_equal_tids(x, y);
#else
# 2160 "beam/erl_threads.h"
    return 1;
#endif
# 2162 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2174 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2180 "beam/erl_threads.h"
#endif
# 2181 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
		    int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2194 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
#endif
# 2200 "beam/erl_threads.h"
#endif
# 2201 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2214 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2220 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2224 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2227 "beam/erl_threads.h"
#endif
# 2228 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2240 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2243 "beam/erl_threads.h"
#endif
# 2244 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2256 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2259 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2263 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2266 "beam/erl_threads.h"
#endif
# 2267 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_destroy(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&mtx->lc);
#endif
# 2277 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&mtx->lcnt);
#endif
# 2280 "beam/erl_threads.h"
    res = ethr_mutex_destroy(&mtx->mtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy mutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2290 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy mutex");
    }
#endif
# 2293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2299 "beam/erl_threads.h"
erts_mtx_trylock(erts_mtx_t *mtx)
#endif
# 2301 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy(&mtx->lc))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2310 "beam/erl_threads.h"

    res = ethr_mutex_trylock(&mtx->mtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_x(res == 0, &mtx->lc,file,line);
#else
# 2317 "beam/erl_threads.h"
    erts_lc_trylock(res == 0, &mtx->lc);
#endif
# 2319 "beam/erl_threads.h"
#endif
# 2320 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, res);
#endif    
# 2323 "beam/erl_threads.h"
    return res;
#else
# 2325 "beam/erl_threads.h"
    return 0;
#endif
# 2327 "beam/erl_threads.h"

}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2334 "beam/erl_threads.h"
erts_mtx_lock(erts_mtx_t *mtx)
#endif
# 2336 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&mtx->lc, file, line);
#else
# 2342 "beam/erl_threads.h"
    erts_lc_lock(&mtx->lc);
#endif
# 2344 "beam/erl_threads.h"
#endif
# 2345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2348 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
#endif
# 2352 "beam/erl_threads.h"
#endif
# 2353 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_unlock(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2362 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2365 "beam/erl_threads.h"
    ethr_mutex_unlock(&mtx->mtx);
#endif
# 2367 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_mtx_is_locked(erts_mtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2379 "beam/erl_threads.h"
    return 0;
#endif
# 2381 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_init(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_init(cnd);
    if (res)
	erts_thr_fatal_error(res, "initialize condition variable");
#endif
# 2391 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_destroy(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_destroy(cnd);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy cond destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2407 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy condition variable");
    }
#endif
# 2410 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2420 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2423 "beam/erl_threads.h"
    res = ethr_cond_wait(cnd, &mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock(&mtx->lc);
#endif
# 2427 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2430 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post(&mtx->lcnt);
#endif
# 2433 "beam/erl_threads.h"
    if (res != 0 && res != EINTR)
	erts_thr_fatal_error(res, "wait on condition variable");
#endif
# 2436 "beam/erl_threads.h"
}

/*
 * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_cnd_signal(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_signal(cnd);
#endif
# 2454 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_cnd_broadcast(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_broadcast(cnd);
#endif
# 2463 "beam/erl_threads.h"
}

/* rwmutex */

ERTS_GLB_INLINE void
erts_rwmtx_set_reader_group(int no)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2475 "beam/erl_threads.h"
    res = ethr_rwmutex_set_reader_group(no);
    if (res != 0)
	erts_thr_fatal_error(res, "set reader group");
#endif
# 2479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
		      erts_rwmtx_opt_t *opt,
		      char *name,
		      Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
#endif
# 2494 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (name && name[0] == '\0')
      erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
    else
      erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
#endif
# 2500 "beam/erl_threads.h"
#endif
# 2501 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
		  char *name,
		  Eterm extra)
{
    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
		    erts_rwmtx_opt_t *opt,
		    char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2523 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
#endif
# 2526 "beam/erl_threads.h"
#endif
# 2527 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
{
    erts_rwmtx_init_opt(rwmtx, NULL, name);
}

ERTS_GLB_INLINE void
erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&rwmtx->lc);
#endif
# 2543 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&rwmtx->lcnt);
#endif
# 2546 "beam/erl_threads.h"
    res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwmutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2556 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwmutex");
    }
#endif
# 2559 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2565 "beam/erl_threads.h"
erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
#endif
# 2567 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2576 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2583 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2585 "beam/erl_threads.h"
#endif
# 2586 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
#endif
# 2589 "beam/erl_threads.h"
    
    return res;
#else
# 2592 "beam/erl_threads.h"
    return 0;
#endif
# 2594 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2600 "beam/erl_threads.h"
erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
#endif
# 2602 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2608 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2610 "beam/erl_threads.h"
#endif
# 2611 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2614 "beam/erl_threads.h"
    ethr_rwmutex_rlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2618 "beam/erl_threads.h"
#endif
# 2619 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2628 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2631 "beam/erl_threads.h"
    ethr_rwmutex_runlock(&rwmtx->rwmtx);
#endif
# 2633 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2640 "beam/erl_threads.h"
erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
#endif
# 2642 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2651 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2658 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2660 "beam/erl_threads.h"
#endif
# 2661 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2664 "beam/erl_threads.h"
    
    return res;
#else
# 2667 "beam/erl_threads.h"
    return 0;
#endif
# 2669 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2675 "beam/erl_threads.h"
erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
#endif
# 2677 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2683 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2685 "beam/erl_threads.h"
#endif
# 2686 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2689 "beam/erl_threads.h"
    ethr_rwmutex_rwlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2693 "beam/erl_threads.h"
#endif
# 2694 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2703 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2706 "beam/erl_threads.h"
    ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
#endif
# 2708 "beam/erl_threads.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_rwmtx_trywlock(erts_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wlock(erts_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx)
{

}

#endif
# 2735 "beam/erl_threads.h"

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2746 "beam/erl_threads.h"
    return 0;
#endif
# 2748 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2760 "beam/erl_threads.h"
    return 0;
#endif
# 2762 "beam/erl_threads.h"
}

/* No atomic ops */

ERTS_GLB_INLINE void
erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    var->sint[0] = val->sint[0];
    var->sint[1] = val->sint[1];
}

ERTS_GLB_INLINE void
erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    val->sint[0] = var->sint[0];
    val->sint[1] = var->sint[1];
}

ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *new_val,
					      erts_no_dw_atomic_t *old_val)
{
    if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
	erts_no_dw_atomic_read(var, old_val);
	return 0;
    }
    else {
	erts_no_dw_atomic_set(var, new_val);
	return !0;
    }
}

ERTS_GLB_INLINE void
erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read(erts_no_atomic_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_inc_read(erts_no_atomic_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_dec_read(erts_no_atomic_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic_inc(erts_no_atomic_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic_dec(erts_no_atomic_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
{
    erts_aint_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
		       erts_aint_t new,
		       erts_aint_t expected)
{
    erts_aint_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bset(erts_no_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    erts_aint_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic32 */

ERTS_GLB_INLINE void
erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read(erts_no_atomic32_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_inc(erts_no_atomic32_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_dec(erts_no_atomic32_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
{
    erts_aint32_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
			 erts_aint32_t new,
			 erts_aint32_t expected)
{
    erts_aint32_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    erts_aint32_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic64 */

ERTS_GLB_INLINE void
erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read(erts_no_atomic64_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_inc(erts_no_atomic64_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_dec(erts_no_atomic64_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
{
    erts_aint64_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
			 erts_aint64_t new,
			 erts_aint64_t expected)
{
    erts_aint64_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
			   erts_aint64_t mask,
			   erts_aint64_t set)
{
    erts_aint64_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* spinlock */

ERTS_GLB_INLINE void
erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3098 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
#endif
# 3101 "beam/erl_threads.h"
#else
# 3102 "beam/erl_threads.h"
    (void)lock;
#endif
# 3104 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
			 Uint16 opt)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3117 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
#endif
# 3120 "beam/erl_threads.h"
#else
# 3121 "beam/erl_threads.h"
    (void)lock;
#endif
# 3123 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_spinlock_init(erts_spinlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK);
#endif
# 3136 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
#endif
# 3139 "beam/erl_threads.h"
#else
# 3140 "beam/erl_threads.h"
    (void)lock;
#endif
# 3142 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_destroy(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3152 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3155 "beam/erl_threads.h"
    res = ethr_spinlock_destroy(&lock->slck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy spinlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3165 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3168 "beam/erl_threads.h"
    (void)lock;
#endif
# 3170 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spin_unlock(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&lock->lc);
#endif
# 3179 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&lock->lcnt);
#endif
# 3182 "beam/erl_threads.h"
    ethr_spin_unlock(&lock->slck);
#else
# 3184 "beam/erl_threads.h"
    (void)lock;
#endif
# 3186 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line)
#else
# 3192 "beam/erl_threads.h"
erts_spin_lock(erts_spinlock_t *lock)
#endif
# 3194 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&lock->lc,file,line);
#else
# 3200 "beam/erl_threads.h"
    erts_lc_lock(&lock->lc);
#endif
# 3202 "beam/erl_threads.h"
#endif
# 3203 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&lock->lcnt);
#endif
# 3206 "beam/erl_threads.h"
    ethr_spin_lock(&lock->slck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3210 "beam/erl_threads.h"
#else
# 3211 "beam/erl_threads.h"
    (void)lock;
#endif
# 3213 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3225 "beam/erl_threads.h"
    return 0;
#endif
# 3227 "beam/erl_threads.h"
}

/* rwspinlock */

ERTS_GLB_INLINE void
erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
#endif
# 3241 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
#endif
# 3244 "beam/erl_threads.h"
#else
# 3245 "beam/erl_threads.h"
    (void)lock;
#endif
# 3247 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_init(erts_rwlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK);
#endif
# 3259 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
#endif
# 3262 "beam/erl_threads.h"
#else
# 3263 "beam/erl_threads.h"
    (void)lock;
#endif
# 3265 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_destroy(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3275 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3278 "beam/erl_threads.h"
    res = ethr_rwlock_destroy(&lock->rwlck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3288 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3291 "beam/erl_threads.h"
    (void)lock;
#endif
# 3293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_read_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3302 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3305 "beam/erl_threads.h"
    ethr_read_unlock(&lock->rwlck);
#else
# 3307 "beam/erl_threads.h"
    (void)lock;
#endif
# 3309 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3315 "beam/erl_threads.h"
erts_read_lock(erts_rwlock_t *lock)
#endif
# 3317 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 3323 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3325 "beam/erl_threads.h"
#endif
# 3326 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3329 "beam/erl_threads.h"
    ethr_read_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3333 "beam/erl_threads.h"
#else
# 3334 "beam/erl_threads.h"
    (void)lock;
#endif
# 3336 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_write_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3348 "beam/erl_threads.h"
    ethr_write_unlock(&lock->rwlck);
#else
# 3350 "beam/erl_threads.h"
    (void)lock;
#endif
# 3352 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3358 "beam/erl_threads.h"
erts_write_lock(erts_rwlock_t *lock)
#endif
# 3360 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 3366 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3368 "beam/erl_threads.h"
#endif
# 3369 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3372 "beam/erl_threads.h"
    ethr_write_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3376 "beam/erl_threads.h"
#else
# 3377 "beam/erl_threads.h"
    (void)lock;
#endif
# 3379 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3391 "beam/erl_threads.h"
    return 0;
#endif
# 3393 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3405 "beam/erl_threads.h"
    return 0;
#endif
# 3407 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_create(keyp, keyname);
    if (res)
	erts_thr_fatal_error(res, "create thread specific data key");
#endif
# 3417 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_delete(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_delete(key);
    if (res)
	erts_thr_fatal_error(res, "delete thread specific data key");
#endif
# 3427 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_set(erts_tsd_key_t key, void *value)
{
#ifdef USE_THREADS
    int res = ethr_tsd_set(key, value);
    if (res)
	erts_thr_fatal_error(res, "set thread specific data");
#endif
# 3437 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void *
erts_tsd_get(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    return ethr_tsd_get(key);
#else
# 3445 "beam/erl_threads.h"
    return NULL;
#endif
# 3447 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
{
#ifdef USE_THREADS
    return (erts_tse_t *) ethr_get_ts_event();
#else
# 3454 "beam/erl_threads.h"
    return (erts_tse_t *) NULL;
#endif
# 3456 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_leave_ts_event(ep);
#endif
# 3463 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
{
#ifdef USE_THREADS
    int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
    if (res != 0)
	erts_thr_fatal_error(res, "prepare timed");
#endif
# 3472 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_set(&((ethr_ts_event *) ep)->event);
#endif
# 3479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_reset(&((ethr_ts_event *) ep)->event);
#endif
# 3486 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
#else
# 3493 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3495 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
{
#ifdef USE_THREADS
    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
#else
# 3502 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3504 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_twait(&((ethr_ts_event *) ep)->event,
			    (ethr_sint64_t) tmo);
#else
# 3512 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3514 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_stwait(&((ethr_ts_event *) ep)->event,
			     spincount,
			     (ethr_sint64_t) tmo);
#else
# 3523 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3525 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
#else
# 3532 "beam/erl_threads.h"
    return 0;
#endif
# 3534 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
{
#ifdef USE_THREADS
    int res = ethr_set_main_thr_status(on, no);
    if (res != 0)
	erts_thr_fatal_error(res, "set thread main status");
#endif
# 3543 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_thr_get_main_status(void)
{
#ifdef USE_THREADS
    int main_status;
    int res = ethr_get_main_thr_status(&main_status);
    if (res != 0)
	erts_thr_fatal_error(res, "get thread main status");
    return main_status;
#else
# 3554 "beam/erl_threads.h"
    return 1;
#endif
# 3556 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_yield(void)
{
#ifdef USE_THREADS
    int res = ETHR_YIELD();
    if (res != 0)
	erts_thr_fatal_error(res, "yield");
#endif    
# 3565 "beam/erl_threads.h"
}


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS

ERTS_GLB_INLINE void
erts_thr_kill(erts_tid_t tid, int sig) {
#ifdef USE_THREADS
  int res = ethr_kill((ethr_tid)tid, sig);
  if (res)
    erts_thr_fatal_error(res, "killing thread");
#endif
# 3577 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef USE_THREADS
    int res = ethr_sigmask(how, set, oset);
    if (res)
	erts_thr_fatal_error(res, "get or set signal mask");
#endif
# 3587 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef USE_THREADS
    int res;
    do {
	res = ethr_sigwait(set, sig);
    } while (res == EINTR);
    if (res)
	erts_thr_fatal_error(res, "to wait for signal");
#endif
# 3600 "beam/erl_threads.h"
}

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 3603 "beam/erl_threads.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 3605 "beam/erl_threads.h"

#endif /* #ifndef ERL_THREAD_H__ */
# 3607 "beam/erl_threads.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_atomic_init
#undef erts_atomic_set
#undef erts_atomic_read
#undef erts_atomic_inctest
#undef erts_atomic_dectest
#undef erts_atomic_inc
#undef erts_atomic_dec
#undef erts_atomic_addtest
#undef erts_atomic_add
#undef erts_atomic_xchg
#undef erts_atomic_cmpxchg
#undef erts_atomic_bor
#undef erts_atomic_band

#undef erts_atomic32_init
#undef erts_atomic32_set
#undef erts_atomic32_read
#undef erts_atomic32_inctest
#undef erts_atomic32_dectest
#undef erts_atomic32_inc
#undef erts_atomic32_dec
#undef erts_atomic32_addtest
#undef erts_atomic32_add
#undef erts_atomic32_xchg
#undef erts_atomic32_cmpxchg
#undef erts_atomic32_bor
#undef erts_atomic32_band

#endif
# 3641 "beam/erl_threads.h"
# 32 "beam/erl_alloc.h" 2
#endif
# 33 "beam/erl_alloc.h"

#ifdef DEBUG
#  undef ERTS_ALC_WANT_INLINE
#  define ERTS_ALC_WANT_INLINE 0
#endif
# 38 "beam/erl_alloc.h"

#ifndef ERTS_ALC_WANT_INLINE
#  define ERTS_ALC_WANT_INLINE 1
#endif
# 42 "beam/erl_alloc.h"

#if ERTS_CAN_INLINE && ERTS_ALC_WANT_INLINE
#  define ERTS_ALC_DO_INLINE 1
#  define ERTS_ALC_INLINE static ERTS_INLINE
#else
# 47 "beam/erl_alloc.h"
#  define ERTS_ALC_DO_INLINE 0
#  define ERTS_ALC_INLINE
#endif
# 50 "beam/erl_alloc.h"

#define ERTS_ALC_NO_FIXED_SIZES \
  (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1)

void erts_sys_alloc_init(void);
void *erts_sys_alloc(ErtsAlcType_t, void *, Uint);
void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint);
void erts_sys_free(ErtsAlcType_t, void *, void *);
#if ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC
/*
 * Note 'alignment' must remain the same in calls to
 * 'erts_sys_aligned_realloc()' and 'erts_sys_aligned_free()'
 * as in the initial call to 'erts_sys_aligned_alloc()'.
 */
void *erts_sys_aligned_alloc(UWord alignment, UWord size);
void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old_size);
void erts_sys_aligned_free(UWord alignment, void *ptr);
#endif
# 68 "beam/erl_alloc.h"

Eterm erts_memory(int *, void *, void *, Eterm);
Eterm erts_allocated_areas(int *, void *, void *);

Eterm erts_alloc_util_allocators(void *proc);
void erts_allocator_info(int, void *);
Eterm erts_allocator_options(void *proc);

struct process;

int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs,
			    int only_sz, int internal);

#define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0}
typedef struct {
    int ncpu;
} ErtsAllocInitOpts;

typedef struct {
    Allctr_t *deallctr[ERTS_ALC_A_MAX+1];
    int pref_ix[ERTS_ALC_A_MAX+1];
    int flist_ix[ERTS_ALC_A_MAX+1];
    int pre_alc_ix;
} ErtsSchedAllocData;

void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop);
void erts_alloc_late_init(void);

#if defined(GET_ERTS_ALC_TEST) || defined(ERTS_ALC_INTERNAL__)
/* Only for testing */
UWord erts_alc_test(UWord,
		    UWord,
		    UWord,
		    UWord);
#endif
# 103 "beam/erl_alloc.h"

#define ERTS_ALC_O_ALLOC		0
#define ERTS_ALC_O_REALLOC		1
#define ERTS_ALC_O_FREE			2

#define ERTS_ALC_E_NOTSUP		0
#define ERTS_ALC_E_NOMEM		1
#define ERTS_ALC_E_NOALLCTR		2

#define ERTS_ALC_MIN_LONG_LIVED_TIME	(10*60*1000)

typedef struct {
    int alloc_util;
    int enabled;
    int thr_spec;
    void *extra;
} ErtsAllocatorInfo_t;

typedef struct {
    void *	(*alloc)	(ErtsAlcType_t, void *, Uint);
    void *	(*realloc)	(ErtsAlcType_t, void *, void *, Uint);
    void	(*free)		(ErtsAlcType_t, void *, void *);
    void *extra;
} ErtsAllocatorFunctions_t;

extern ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1];
extern ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1];

typedef struct {
    int enabled;
    int dd;
    int aix;
    int size;
    Allctr_t **allctr;
} ErtsAllocatorThrSpec_t;

extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];

typedef struct ErtsAllocatorWrapper_t_ {
    void (*lock)(void);
    void (*unlock)(void);
    struct ErtsAllocatorWrapper_t_* next;
}ErtsAllocatorWrapper_t;
ErtsAllocatorWrapper_t *erts_allctr_wrappers;
extern int erts_allctr_wrapper_prelocked;
extern erts_tsd_key_t erts_allctr_prelock_tsd_key;
void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper);
void erts_allctr_wrapper_pre_lock(void);
void erts_allctr_wrapper_pre_unlock(void);

void erts_alloc_register_scheduler(void *vesdp);
#ifdef ERTS_SMP
void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
						 int *need_thr_progress,
						 ErtsThrPrgrVal *thr_prgr_p,
						 int *more_work);
#endif
# 160 "beam/erl_alloc.h"
erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);

__decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)		
     __noreturn;
__decl_noreturn void erts_alloc_n_enomem(ErtsAlcType_t,Uint)		
     __noreturn;
__decl_noreturn void erts_realloc_enomem(ErtsAlcType_t,void*,Uint)	
     __noreturn;
__decl_noreturn void erts_realloc_n_enomem(ErtsAlcType_t,void*,Uint)	
     __noreturn;
__decl_noreturn void erts_alc_fatal_error(int,int,ErtsAlcType_t,...)	
     __noreturn;

/* --- DO *NOT* USE THESE DEPRECATED FUNCTIONS ---    Instead use:       */
void *safe_alloc(Uint)               __deprecated; /* erts_alloc()       */
void *safe_realloc(void *, Uint)     __deprecated; /* erts_realloc()     */
void  sys_free(void *)               __deprecated; /* erts_free()        */
void *sys_alloc(Uint )               __deprecated; /* erts_alloc_fnf()   */
void *sys_realloc(void *, Uint)      __deprecated; /* erts_realloc_fnf() */

/*
 * erts_alloc[_fnf](), erts_realloc[_fnf](), erts_free() works as
 * malloc(), realloc(), and free() with the following exceptions:
 *
 * * They take an extra type argument as first argument which is
 *   the memory type to operate on. Memory types are generated
 *   (as ERTS_ALC_T_[SOMETHING] defines) from the erl_alloc.types
 *   configuration file.
 * * The erts_alloc() and erts_realloc() functions terminate the
 *   emulator if memory cannot be obtained. The _fnf (Failure Not
 *   Fatal) suffixed versions return NULL if memory cannot be
 *   obtained.
 * * They may be static functions so function pointers to "the same"
 *   function may differ.
 *
 * IMPORTANT: Memory allocated or reallocated as type X, can only
 *            be reallocated or deallocated as type X.
 */

#if !ERTS_ALC_DO_INLINE

void *erts_alloc(ErtsAlcType_t type, Uint size);
void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size);
void erts_free(ErtsAlcType_t type, void *ptr);
void *erts_alloc_fnf(ErtsAlcType_t type, Uint size);
void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size);
int erts_is_allctr_wrapper_prelocked(void);

#endif /* #if !ERTS_ALC_DO_INLINE */
# 209 "beam/erl_alloc.h"

void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size);

#ifndef ERTS_CACHE_LINE_SIZE
/* Assumed cache line size */
#  define ERTS_CACHE_LINE_SIZE ((UWord) ASSUMED_CACHE_LINE_SIZE)
#  define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1)
#endif
# 217 "beam/erl_alloc.h"

#if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__)

ERTS_ALC_INLINE
void *erts_alloc(ErtsAlcType_t type, Uint size)
{
    void *res;
    res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
	ERTS_ALC_T2N(type),
	erts_allctrs[ERTS_ALC_T2A(type)].extra,
	size);
    if (!res)
	erts_alloc_n_enomem(ERTS_ALC_T2N(type), size);
    return res;
}

ERTS_ALC_INLINE
void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size)
{
    void *res;
    res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
	ERTS_ALC_T2N(type),
	erts_allctrs[ERTS_ALC_T2A(type)].extra,
	ptr,
	size);
    if (!res)
	erts_realloc_n_enomem(ERTS_ALC_T2N(type), ptr, size);
    return res;
}

ERTS_ALC_INLINE
void erts_free(ErtsAlcType_t type, void *ptr)
{
    (*erts_allctrs[ERTS_ALC_T2A(type)].free)(
	ERTS_ALC_T2N(type),
	erts_allctrs[ERTS_ALC_T2A(type)].extra,
	ptr);
}


ERTS_ALC_INLINE
void *erts_alloc_fnf(ErtsAlcType_t type, Uint size)
{
    return (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)(
	ERTS_ALC_T2N(type),
	erts_allctrs[ERTS_ALC_T2A(type)].extra,
	size);
}


ERTS_ALC_INLINE
void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size)
{
    return (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)(
	ERTS_ALC_T2N(type),
	erts_allctrs[ERTS_ALC_T2A(type)].extra,
	ptr,
	size);
}

ERTS_ALC_INLINE
int erts_is_allctr_wrapper_prelocked(void)
{
    return erts_allctr_wrapper_prelocked                 /* locked */
	&& !!erts_tsd_get(erts_allctr_prelock_tsd_key);  /* by me  */
}

#endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */
# 285 "beam/erl_alloc.h"

#define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id())

typedef void (*erts_alloc_verify_func_t)(Allctr_t *);

erts_alloc_verify_func_t
erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr);

#define ERTS_ALC_DATA_ALIGN_SIZE(SZ) \
  (((((SZ) - 1) / 8) + 1) * 8)

#define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \
  (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE)

#define ERTS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
		      (void) 0, (void) 0, (void) 0)

#define ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
static erts_smp_spinlock_t NAME##_lck;					\
ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
		      erts_smp_spinlock_init(&NAME##_lck, #NAME "_alloc_lock"),\
		      erts_smp_spin_lock(&NAME##_lck),			\
		      erts_smp_spin_unlock(&NAME##_lck))

#ifdef ERTS_SMP

#define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
ERTS_SMP_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)

#else /* !ERTS_SMP */
# 316 "beam/erl_alloc.h"

#define ERTS_TS_QUALLOC_IMPL(NAME, TYPE, PASZ, ALCT)			\
static erts_mtx_t NAME##_lck;						\
ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT,				\
		      erts_mtx_init(NAME##_lck, #NAME "_alloc_lock"),	\
		      erts_mtx_lock(&NAME##_lck),			\
		      erts_mtx_unlock(&NAME##_lck))


#endif
# 326 "beam/erl_alloc.h"

#define ERTS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0)

#define ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ)				\
static erts_spinlock_t NAME##_lck;					\
ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ,					\
		    erts_spinlock_init(&NAME##_lck, #NAME "_alloc_lock"),\
		    erts_spin_lock(&NAME##_lck),			\
		    erts_spin_unlock(&NAME##_lck))

#ifdef ERTS_SMP

#define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ)				\
  ERTS_TS_PALLOC_IMPL(NAME, TYPE, PASZ)

#else /* !ERTS_SMP */
# 343 "beam/erl_alloc.h"

#define ERTS_SMP_PALLOC_IMPL(NAME, TYPE, PASZ)				\
  ERTS_PALLOC_IMPL(NAME, TYPE, PASZ)

#endif
# 348 "beam/erl_alloc.h"

#define ERTS_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT, ILCK, LCK, ULCK)	\
ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, ILCK, LCK, ULCK)		\
static void								\
init_##NAME##_alloc(void)						\
{									\
    init_##NAME##_pre_alloc();						\
}									\
static ERTS_INLINE TYPE *						\
NAME##_alloc(void)							\
{									\
    TYPE *res = NAME##_pre_alloc();					\
    if (!res)								\
	res = erts_alloc(ALCT, sizeof(TYPE));				\
    return res;								\
}									\
static ERTS_INLINE void							\
NAME##_free(TYPE *p)							\
{									\
    if (!NAME##_pre_free(p))						\
	erts_free(ALCT, (void *) p);					\
}

#ifdef ERTS_SMP
#define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ)			\
  ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)
#else
# 375 "beam/erl_alloc.h"
#define ERTS_SCHED_PREF_PALLOC_IMPL(NAME, TYPE, PASZ)			\
  ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
#endif
# 378 "beam/erl_alloc.h"

#ifdef ERTS_SMP
#define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ)				\
ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ)
#else
# 383 "beam/erl_alloc.h"
#define ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ)				\
ERTS_PRE_ALLOC_IMPL(NAME##_pre, TYPE, PASZ, (void) 0, (void) 0, (void) 0)
#endif
# 386 "beam/erl_alloc.h"

#define ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(NAME, TYPE, PASZ, ALCT)	\
ERTS_SCHED_PREF_AUX(NAME, TYPE, PASZ)					\
static void								\
init_##NAME##_alloc(void)						\
{									\
    init_##NAME##_pre_alloc();						\
}									\
static ERTS_INLINE TYPE *						\
NAME##_alloc(void)							\
{									\
    TYPE *res = NAME##_pre_alloc();					\
    if (!res)								\
	res = erts_alloc(ALCT, sizeof(TYPE));				\
    return res;								\
}									\
static ERTS_INLINE void							\
NAME##_free(TYPE *p)							\
{									\
    if (!NAME##_pre_free(p))						\
	erts_free(ALCT, (void *) p);					\
}

#ifdef DEBUG
#define ERTS_PRE_ALLOC_SIZE(SZ) 2
#define ERTS_PRE_ALLOC_CLOBBER(P, T) memset((void *) (P), 0xfd, sizeof(T))
#else
# 413 "beam/erl_alloc.h"
#define ERTS_PRE_ALLOC_SIZE(SZ) ((SZ) > 1 ? (SZ) : 1)
#define ERTS_PRE_ALLOC_CLOBBER(P, T)
#endif
# 416 "beam/erl_alloc.h"

#define ERTS_PRE_ALLOC_IMPL(NAME, TYPE, PASZ, ILCK, LCK, ULCK)		\
union erts_qa_##NAME##__ {						\
    TYPE type;								\
    union erts_qa_##NAME##__ *next;					\
};									\
static union erts_qa_##NAME##__						\
    qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))];			\
static union erts_qa_##NAME##__ *qa_freelist_##NAME;			\
static void								\
init_##NAME##_alloc(void)						\
{									\
    int i;								\
    qa_freelist_##NAME = &qa_prealcd_##NAME[0];				\
    for (i = 1; i < ERTS_PRE_ALLOC_SIZE((PASZ)); i++) {			\
	ERTS_PRE_ALLOC_CLOBBER(&qa_prealcd_##NAME[i-1],			\
			       union erts_qa_##NAME##__);		\
	qa_prealcd_##NAME[i-1].next = &qa_prealcd_##NAME[i];		\
    }									\
    ERTS_PRE_ALLOC_CLOBBER(&qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))-1],\
			   union erts_qa_##NAME##__);			\
    qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))-1].next = NULL;	\
    ILCK;								\
}									\
static ERTS_INLINE TYPE *						\
NAME##_alloc(void)							\
{									\
    TYPE *res;								\
    LCK;								\
    if (!qa_freelist_##NAME)						\
	res = NULL;							\
    else {								\
	res = &qa_freelist_##NAME->type;				\
	qa_freelist_##NAME = qa_freelist_##NAME->next;			\
    }									\
    ULCK;								\
    return res;								\
}									\
static ERTS_INLINE int							\
NAME##_free(TYPE *p)							\
{									\
    union erts_qa_##NAME##__ * up;					\
    up = ((union erts_qa_##NAME##__ *)					\
	  (((char *) p)							\
	   - ((char *) &((union erts_qa_##NAME##__ *) 0)->type)));	\
    if (up > &qa_prealcd_##NAME[ERTS_PRE_ALLOC_SIZE((PASZ))-1]		\
	|| up < &qa_prealcd_##NAME[0])					\
	return 0;							\
    else {								\
	LCK;								\
	ERTS_PRE_ALLOC_CLOBBER(up, union erts_qa_##NAME##__);		\
	up->next = qa_freelist_##NAME;					\
	qa_freelist_##NAME = up;					\
	ULCK;								\
	return 1;							\
    }									\
}

#if 0 /* expanded by -frewrite-includes */
#include "erl_sched_spec_pre_alloc.h"
#endif /* expanded by -frewrite-includes */
# 474 "beam/erl_alloc.h"
# 1 "beam/erl_sched_spec_pre_alloc.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Scheduler specific pre-allocators. Each scheduler
 *              thread allocates memory in its own private chunk of
 *              memory. Memory blocks deallocated by remote
 *              schedulers (or other threads) are passed back to
 *              the chunk owner via a lock-free data structure.
 *
 * Author: 	Rickard Green
 */

#ifndef ERTS_SCHED_SPEC_PRE_ALLOC_H__
#define ERTS_SCHED_SPEC_PRE_ALLOC_H__

#ifdef ERTS_SMP

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 38 "beam/erl_sched_spec_pre_alloc.h"
# 1 "beam/erl_thr_progress.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread progress information. Used by lock free algorithms
 *              to determine when all involved threads are guaranteed to
 *              have passed a specific point of execution.
 *
 *              Usage instructions can be found in ert_thr_progress.c
 *
 * Author: 	Rickard Green
 */

#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
#define ERL_THR_PROGRESS_H__TSD_TYPE__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/erl_thr_progress.h"
# 35 "beam/erl_thr_progress.h"

#ifndef ERTS_SMP

#define erts_smp_thr_progress_block() ((void) 0)
#define erts_smp_thr_progress_unblock() ((void) 0)
#define erts_smp_thr_progress_is_blocking() 1

#else /* ERTS_SMP */
# 43 "beam/erl_thr_progress.h"

#define erts_smp_thr_progress_block erts_thr_progress_block
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking

void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);

typedef Uint64 ErtsThrPrgrVal;

#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */

typedef struct {
    ErtsThrPrgrVal next;
    ErtsThrPrgrVal current;
    int chk_next_ix;
    struct {
	int current;
	int waiting;
    } umrefc_ix;
} ErtsThrPrgrLeaderState;

typedef struct {
    int id;
    int is_managed;
    int is_blocking;
#ifdef ERTS_ENABLE_LOCK_CHECK
    int is_delaying; /* managed is always delaying */
#endif
# 73 "beam/erl_thr_progress.h"
    int is_temporary;

    /* --- Part below only for registered threads --- */

    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];

    /* --- Part below only for managed threads --- */

    int leader; /* Needs to be first in the managed threads part */
    int active;
    ErtsThrPrgrVal confirmed;
    ErtsThrPrgrLeaderState leader_state;
} ErtsThrPrgrData;

int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
void erts_thr_progress_fatal_error_wait(SWord timeout);

#endif /* ERTS_SMP */
# 91 "beam/erl_thr_progress.h"

typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
struct ErtsThrPrgrLaterOp_ {
#ifdef ERTS_SMP
    ErtsThrPrgrVal later;
#endif
# 97 "beam/erl_thr_progress.h"
    void (*func)(void *);
    void *data;
    ErtsThrPrgrLaterOp *next;
};

#endif
# 103 "beam/erl_thr_progress.h"

#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
#define ERL_THR_PROGRESS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 107 "beam/erl_thr_progress.h"
# 108 "beam/erl_thr_progress.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 108 "beam/erl_thr_progress.h"
# 109 "beam/erl_thr_progress.h"

#ifdef ERTS_SMP

/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))

extern erts_tsd_key_t erts_thr_prgr_data_key__;

#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t

typedef struct {
    void *arg;
    void (*wakeup)(void *);
    void (*prepare_wait)(void *);
    void (*wait)(void *);
    void (*finalize_wait)(void *);
} ErtsThrPrgrCallbacks;

typedef struct {
    ERTS_THR_PRGR_ATOMIC current;
} ErtsThrPrgr;

typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)

extern ErtsThrPrgr erts_thr_prgr__;

void erts_thr_progress_pre_init(void);
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
					       ErtsThrPrgrCallbacks *,
					       int);
void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
			      ErtsThrPrgrVal value);
int erts_thr_progress_update(ErtsSchedulerData *esdp);
int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
void erts_thr_progress_unmanaged_continue__(int umrefc_ix);

void erts_thr_progress_dbg_print_state(void);

ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);

ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
#endif
# 168 "beam/erl_thr_progress.h"
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
					    ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
}

ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_managed;
}

ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)
{
    if (erts_thr_progress_is_managed_thread())
	return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
    else
	return erts_thr_progress_unmanaged_delay__();
}

ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
{
    ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
	   || erts_thr_progress_is_managed_thread());
    if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
	erts_thr_progress_unmanaged_continue__(handle);
}

#ifdef ERTS_ENABLE_LOCK_CHECK

ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_delaying;
}

#endif
# 233 "beam/erl_thr_progress.h"

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
{
    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
	return ((ErtsThrPrgrVal) 0);
    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
	return ((ErtsThrPrgrVal) 1);
    else
	return val + ((ErtsThrPrgrVal) 2);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData *esdp)
{
    ErtsThrPrgrData *tpd;
    ErtsThrPrgrVal val;
    if (esdp) {
	tpd = &esdp->thr_progress_data;
    managed_thread:
	val = tpd->confirmed;
	ERTS_THR_MEMORY_BARRIER;
    }
    else {
	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
	if (tpd && tpd->is_managed)
	    goto managed_thread;
	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
    }
    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
    return erts_thr_progress_current_to_later__(val);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
    else
	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}

ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
{
    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
	/* May have wrapped... */
	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
	    /*
	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
	     * these ranges 'current' is considered later than 'val0'.
	     */
	    return 1;
	}
    }
    return val1 > val0;
}

ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
{
    if (this == val)
	return 1;
    return erts_thr_progress_has_passed__(this, val);
}

ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
}

ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    if (val1 == val2)
	return 0;
    if (erts_thr_progress_has_passed__(val1, val2))
	return 1;
    else
	return -1;
}	

ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)
{
    ErtsThrPrgrVal current = erts_thr_progress_current();
    return erts_thr_progress_has_reached_this(current, val);
}

#endif
# 326 "beam/erl_thr_progress.h"

#endif /* ERTS_SMP */
# 328 "beam/erl_thr_progress.h"

#endif
# 330 "beam/erl_thr_progress.h"
# 39 "beam/erl_sched_spec_pre_alloc.h" 2
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

#ifdef DEBUG
#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P)				\
do {									\
    ASSERT((void *) (C) < (void *) (P));				\
    ASSERT((void *) (P)							\
	   < (void *) (((char *) (C)) + (A)->chunks_mem_size));		\
} while (0)
#else
# 49 "beam/erl_sched_spec_pre_alloc.h"
#define ERTS_SPPA_DBG_CHK_IN_CHNK(A, C, P)
#endif
# 51 "beam/erl_sched_spec_pre_alloc.h"

#ifdef DEBUG
extern Uint erts_no_schedulers;
#endif
# 55 "beam/erl_sched_spec_pre_alloc.h"

#define ERTS_SSPA_FORCE_THR_CHECK_PROGRESS 10
#define ERTS_SSPA_MAX_GET_NEW_LOCAL 5

typedef struct {
    char *start;
    char *end;
    int chunks_mem_size;
} erts_sspa_data_t;

typedef union erts_sspa_blk_t_ erts_sspa_blk_t;
union erts_sspa_blk_t_ {
    erts_atomic_t next_atmc;
    erts_sspa_blk_t *next_ptr;
};

typedef struct {
    erts_sspa_blk_t *first;
    erts_sspa_blk_t *last;
    int cnt;
    int lim;
} erts_sspa_local_freelist_t;

typedef struct {
    erts_sspa_blk_t marker;
    erts_atomic_t last;
    erts_atomic_t um_refc[2];
    erts_atomic32_t um_refc_ix;
} erts_sspa_tail_t;

typedef struct {
    /*
     * This structure needs to be cache line aligned for best
     * performance.
     */
    union {
	/* Modified by threads returning memory to this chunk */
	erts_sspa_tail_t data;
	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_sspa_tail_t))];
    } tail;
    /*
     * Everything below this point is *only* accessed by the
     * thread owning this chunk.
     */
    struct {
	int no_thr_progress_check;
	int used_marker;
	erts_sspa_blk_t *first;
	erts_sspa_blk_t *unref_end;
	struct {
	    ErtsThrPrgrVal thr_progress;
	    int thr_progress_reached;
	    int um_refc_ix;
	    erts_sspa_blk_t *unref_end;
	} next;
    } head;
    erts_sspa_local_freelist_t local;
} erts_sspa_chunk_header_t;

typedef struct {
    union {
	erts_sspa_chunk_header_t header;
	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(
		sizeof(erts_sspa_chunk_header_t))];
    } aligned;
    char data[1];
} erts_sspa_chunk_t;

#ifdef DEBUG
ERTS_GLB_INLINE void
check_local_list(erts_sspa_chunk_header_t *chdr);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
check_local_list(erts_sspa_chunk_header_t *chdr)
{
    erts_sspa_blk_t *blk;
    int n = 0;
    for (blk = chdr->local.first; blk; blk = blk->next_ptr)
	n++;
    ASSERT(n == chdr->local.cnt);
}
#endif
# 138 "beam/erl_sched_spec_pre_alloc.h"
#define ERTS_SSPA_DBG_CHK_LCL(CHDR) check_local_list((CHDR))
#else
# 140 "beam/erl_sched_spec_pre_alloc.h"
#define ERTS_SSPA_DBG_CHK_LCL(CHDR)
#endif
# 142 "beam/erl_sched_spec_pre_alloc.h"

erts_sspa_data_t *erts_sspa_create(size_t blk_sz,
				   int pa_size);
void erts_sspa_remote_free(erts_sspa_chunk_header_t *chdr,
			   erts_sspa_blk_t *blk,
			   int cinit);
erts_sspa_blk_t *erts_sspa_process_remote_frees(erts_sspa_chunk_header_t *chdr,
						erts_sspa_blk_t *old_res);

ERTS_GLB_INLINE erts_sspa_chunk_t *erts_sspa_cix2chunk(erts_sspa_data_t *data,
						       int cix);
ERTS_GLB_INLINE int erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr);
ERTS_GLB_INLINE char *erts_sspa_alloc(erts_sspa_data_t *data, int cix);
ERTS_GLB_INLINE int erts_sspa_free(erts_sspa_data_t *data, int cix, char *blk);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_sspa_chunk_t *
erts_sspa_cix2chunk(erts_sspa_data_t *data, int cix)
{
    ASSERT(0 <= cix && cix < erts_no_schedulers);
    return (erts_sspa_chunk_t *) (data->start + cix*data->chunks_mem_size);
}

ERTS_GLB_INLINE int
erts_sspa_ptr2cix(erts_sspa_data_t *data, void *ptr)
{
    int cix;
    size_t diff;
    if ((char *) ptr < data->start || data->end <= (char *) ptr)
	return -1;
    diff = ((char *) ptr) - data->start;
    cix = (int) diff / data->chunks_mem_size;
    ASSERT(0 <= cix && cix < erts_no_schedulers);
    return cix;
}

ERTS_GLB_INLINE char *
erts_sspa_alloc(erts_sspa_data_t *data, int cix)
{
    erts_sspa_chunk_t *chnk;
    erts_sspa_chunk_header_t *chdr;
    erts_sspa_blk_t *res;

    chnk = erts_sspa_cix2chunk(data, cix);
    chdr = &chnk->aligned.header;
    res = chdr->local.first;
    ERTS_SSPA_DBG_CHK_LCL(chdr);
    if (res) {
	ERTS_SSPA_DBG_CHK_LCL(chdr);
	chdr->local.first = res->next_ptr;
	chdr->local.cnt--;
	if (!chdr->local.first)
	    chdr->local.last = NULL;
	ERTS_SSPA_DBG_CHK_LCL(chdr);
    }
    if (chdr->local.cnt <= chdr->local.lim)
	return (char *) erts_sspa_process_remote_frees(chdr, res);
    else if (chdr->head.no_thr_progress_check < ERTS_SSPA_FORCE_THR_CHECK_PROGRESS)
	chdr->head.no_thr_progress_check++;
    ASSERT(res);
    return (char *) res;
}

ERTS_GLB_INLINE int
erts_sspa_free(erts_sspa_data_t *data, int cix, char *cblk)
{
    erts_sspa_chunk_t *chnk;
    erts_sspa_chunk_header_t *chdr;
    erts_sspa_blk_t *blk = (erts_sspa_blk_t *) cblk;
    int chnk_cix = erts_sspa_ptr2cix(data, blk);

    if (chnk_cix < 0)
	return 0;

    chnk = erts_sspa_cix2chunk(data, chnk_cix);
    chdr = &chnk->aligned.header;
    if (chnk_cix != cix) {
	/* Remote chunk */
	erts_sspa_remote_free(chdr, blk, chnk_cix - cix);
    }
    else {
	/* Local chunk */
	ERTS_SSPA_DBG_CHK_LCL(chdr);
	blk->next_ptr = chdr->local.first;
	chdr->local.first = blk;
	if (!chdr->local.last)
	    chdr->local.last = blk;
	chdr->local.cnt++;
	ERTS_SSPA_DBG_CHK_LCL(chdr);
    }

    return 1;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 238 "beam/erl_sched_spec_pre_alloc.h"

#endif /* ERTS_SMP */
# 240 "beam/erl_sched_spec_pre_alloc.h"

#endif /* ERTS_SCHED_SPEC_PRE_ALLOC_H__ */
# 242 "beam/erl_sched_spec_pre_alloc.h"
# 475 "beam/erl_alloc.h" 2

#define ERTS_SCHED_PREF_PRE_ALLOC_IMPL(NAME, TYPE, PASZ)		\
union erts_sspa_##NAME##__ {						\
    erts_sspa_blk_t next;						\
    TYPE type;								\
};									\
									\
static erts_sspa_data_t *sspa_data_##NAME##__;				\
									\
static void								\
init_##NAME##_alloc(void)						\
{									\
    sspa_data_##NAME##__ =						\
	erts_sspa_create(sizeof(union erts_sspa_##NAME##__),		\
			 ERTS_PRE_ALLOC_SIZE((PASZ)));			\
}									\
									\
static TYPE *								\
NAME##_alloc(void)							\
{									\
    ErtsSchedulerData *esdp = erts_get_scheduler_data();		\
    if (!esdp || ERTS_SCHEDULER_IS_DIRTY(esdp))				\
	return NULL;							\
    return (TYPE *) erts_sspa_alloc(sspa_data_##NAME##__,		\
				    (int) esdp->no - 1);		\
}									\
									\
static int								\
NAME##_free(TYPE *p)							\
{									\
    ErtsSchedulerData *esdp = erts_get_scheduler_data();		\
    return erts_sspa_free(sspa_data_##NAME##__,				\
			  esdp ? (int) esdp->no - 1 : -1,		\
			  (char *) p);					\
}

#ifdef DEBUG
#define ERTS_ALC_DBG_BLK_SZ(PTR) (*(((UWord *) (PTR)) - 2))
#endif /* #ifdef DEBUG */
# 514 "beam/erl_alloc.h"

#undef ERTS_ALC_INLINE
#undef ERTS_ALC_ATTRIBUTES

#endif /* #ifndef ERL_ALLOC_H__ */
# 519 "beam/erl_alloc.h"


# 27 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/global.h"
# 28 "beam/global.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/global.h"
# 1 "beam/erl_node_container_utils.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_NODE_CONTAINER_UTILS_H__
#define ERL_NODE_CONTAINER_UTILS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_ptab.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_node_container_utils.h"
# 1 "beam/erl_ptab.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description:	Process/Port table implementation.
 *
 * Author: 	Rickard Green
 */

#ifndef ERL_PTAB_H__
#define ERL_PTAB_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_ptab.h"
# 31 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_ptab.h"
# 32 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_ptab.h"
# 1 "beam/erl_time.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2006-2011. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef ERL_TIME_H__
#define ERL_TIME_H__

/* timer wheel size NEED to be a power of 2 */
#ifdef SMALL_MEMORY
#define ERTS_TIW_SIZE (1 << 13)
#else
# 28 "beam/erl_time.h"
#define ERTS_TIW_SIZE (1 << 16)
#endif
# 30 "beam/erl_time.h"

#if defined(DEBUG) || 0
#define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B)
#else
# 34 "beam/erl_time.h"
#define ERTS_TIME_ASSERT(B) ((void) 1)
#endif
# 36 "beam/erl_time.h"

typedef enum {
    ERTS_NO_TIME_WARP_MODE,
    ERTS_SINGLE_TIME_WARP_MODE,
    ERTS_MULTI_TIME_WARP_MODE
} ErtsTimeWarpMode;

typedef struct ErtsTimerWheel_ ErtsTimerWheel;
typedef ErtsMonotonicTime * ErtsNextTimeoutRef;

extern SysTimeval erts_first_emu_time;


void erts_monitor_time_offset(Eterm id, Eterm ref);
int erts_demonitor_time_offset(Eterm ref);

int erts_init_time_sup(int, ErtsTimeWarpMode);
void erts_late_init_time_sup(void);

ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *);
void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode);
void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime);
Uint erts_timer_wheel_memory_size(void);

#ifdef DEBUG
void erts_p_slpq(void);
#endif
# 63 "beam/erl_time.h"

/* time_sup */

#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
#  ifndef HAVE_ERTS_NOW_CPU
#    define HAVE_ERTS_NOW_CPU
#    ifdef HAVE_GETHRVTIME
#      define erts_start_now_cpu() sys_start_hrvtime()
#      define erts_stop_now_cpu()  sys_stop_hrvtime()
#    endif
# 73 "beam/erl_time.h"
#  endif
# 74 "beam/erl_time.h"
void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
#endif
# 76 "beam/erl_time.h"

typedef UWord erts_approx_time_t;
erts_approx_time_t erts_get_approx_time(void);

int erts_has_time_correction(void);
int erts_check_time_adj_support(int time_correction,
				ErtsTimeWarpMode time_warp_mode);

ErtsTimeWarpMode erts_time_warp_mode(void);

typedef enum {
    ERTS_TIME_OFFSET_PRELIMINARY,
    ERTS_TIME_OFFSET_FINAL,
    ERTS_TIME_OFFSET_VOLATILE
} ErtsTimeOffsetState;

ErtsTimeOffsetState erts_time_offset_state(void); 
ErtsTimeOffsetState erts_finalize_time_offset(void); 
struct process;
Eterm erts_get_monotonic_start_time(struct process *c_p);
Eterm erts_get_monotonic_end_time(struct process *c_p);
Eterm erts_monotonic_time_source(struct process*c_p);
Eterm erts_system_time_source(struct process*c_p);

#ifdef SYS_CLOCK_RESOLUTION
#define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000))
#else
# 103 "beam/erl_time.h"
#define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution)
#endif
# 105 "beam/erl_time.h"

#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000))

struct erts_time_sup_read_only__ {
    ErtsMonotonicTime monotonic_time_unit;
#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
    ErtsMonotonicTime start;
    struct {
	ErtsMonotonicTime native;
	ErtsMonotonicTime nsec;
	ErtsMonotonicTime usec;
	ErtsMonotonicTime msec;
	ErtsMonotonicTime sec;
    } start_offset;
#endif
# 120 "beam/erl_time.h"
#ifndef SYS_CLOCK_RESOLUTION
    ErtsMonotonicTime clktck_resolution;
#endif
# 123 "beam/erl_time.h"
};

typedef struct {
    union {
	struct erts_time_sup_read_only__ o;
	char align__[(((sizeof(struct erts_time_sup_read_only__) - 1)
		       / ASSUMED_CACHE_LINE_SIZE) + 1)
		     * ASSUMED_CACHE_LINE_SIZE];
    } r;
} ErtsTimeSupData;

extern ErtsTimeSupData erts_time_sup__;

ErtsMonotonicTime erts_napi_monotonic_time(int time_unit);
ErtsMonotonicTime erts_napi_time_offset(int time_unit);
ErtsMonotonicTime erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to);

ERTS_GLB_INLINE Uint64
erts_time_unit_conversion(Uint64 value,
			  Uint32 from_time_unit,
			  Uint32 to_time_unit);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Uint64
erts_time_unit_conversion(Uint64 value,
			  Uint32 from_time_unit,
			  Uint32 to_time_unit)
{
    Uint64 high, low, result;
    if (value <= ~((Uint64) 0)/to_time_unit)
	return (value*to_time_unit)/from_time_unit;

    low = value & ((Uint64) 0xffffffff);
    high = (value >> 32) & ((Uint64) 0xffffffff);

    low *= to_time_unit;
    high *= to_time_unit;

    high += (low >> 32) & ((Uint64) 0xffffffff);
    low &= ((Uint64) 0xffffffff);

    result = high % from_time_unit;
    high /= from_time_unit;
    high <<= 32;

    result <<= 32;
    result += low;
    result /= from_time_unit;
    result += high;

    return result;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 178 "beam/erl_time.h"

/*
 * Range of monotonic time internally
 */

#define ERTS_MONOTONIC_BEGIN						\
    ERTS_MONOTONIC_TIME_UNIT
#define ERTS_MONOTONIC_END						\
    ((ERTS_MONOTONIC_TIME_MAX / ERTS_MONOTONIC_TIME_UNIT)		\
     * ERTS_MONOTONIC_TIME_UNIT)

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT

/*
 * If the monotonic time unit is a compile time constant,
 * it is assumed (and need) to be a power of 10.
 */

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT < 1000*1000
#  error Compile time time unit needs to be at least 1000000
#endif
# 199 "beam/erl_time.h"

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000
/* Nano-second time unit */

#define ERTS_MONOTONIC_TO_SEC__(NSEC) ((NSEC) / (1000*1000*1000))
#define ERTS_MONOTONIC_TO_MSEC__(NSEC) ((NSEC) / (1000*1000))
#define ERTS_MONOTONIC_TO_USEC__(NSEC) ((NSEC) / 1000)
#define ERTS_MONOTONIC_TO_NSEC__(NSEC) (NSEC)

#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000*1000))
#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*(1000*1000))
#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))*1000)
#define ERTS_NSEC_TO_MONOTONIC__(NSEC) ((ErtsMonotonicTime) (NSEC))

#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000
# 214 "beam/erl_time.h"
/* Micro-second time unit */

#define ERTS_MONOTONIC_TO_SEC__(USEC) ((USEC) / (1000*1000))
#define ERTS_MONOTONIC_TO_MSEC__(USEC) ((USEC) / 1000)
#define ERTS_MONOTONIC_TO_USEC__(USEC) (USEC)
#define ERTS_MONOTONIC_TO_NSEC__(USEC) ((USEC)*1000)

#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000))
#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*1000)
#define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC))
#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000)

#else
# 227 "beam/erl_time.h"
#error Missing implementation for monotonic time unit
#endif
# 229 "beam/erl_time.h"

#define ERTS_MONOTONIC_TIME_UNIT \
    ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)

/*
 * NOTE! ERTS_MONOTONIC_TIME_START_EXTERNAL *need* to be a multiple
 *       of ERTS_MONOTONIC_TIME_UNIT.
 */

#ifdef ARCH_32
/*
 * Want to use a big-num of arity 2 as long as possible (584 years
 * in the nano-second time unit case).
 */
#define ERTS_MONOTONIC_TIME_START_EXTERNAL		\
    (((((((ErtsMonotonicTime) 1) << 32)-1)	\
       / ERTS_MONOTONIC_TIME_UNIT)		\
      * ERTS_MONOTONIC_TIME_UNIT)		\
     + ERTS_MONOTONIC_TIME_UNIT)

#else /* ARCH_64 */
# 250 "beam/erl_time.h"

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000

/*
 * Using micro second time unit or lower. Start at zero since
 * time will remain an immediate for a very long time anyway
 * (1827 years in the 10 micro second case)...
 */
#define ERTS_MONOTONIC_TIME_START_EXTERNAL ((ErtsMonotonicTime) 0)

#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 10*1000*1000 */
# 261 "beam/erl_time.h"

/*
 * Want to use an immediate as long as possible (36 years in the
 * nano-second time unit case).
*/
#define ERTS_MONOTONIC_TIME_START_EXTERNAL 		\
    ((((ErtsMonotonicTime) MIN_SMALL)		\
      / ERTS_MONOTONIC_TIME_UNIT)		\
     * ERTS_MONOTONIC_TIME_UNIT)

#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
# 272 "beam/erl_time.h"

#endif /* ARCH_64 */
# 274 "beam/erl_time.h"

/*
 * Offsets from internal monotonic time to external monotonic time
 */

#define ERTS_MONOTONIC_OFFSET_NATIVE \
    (ERTS_MONOTONIC_TIME_START_EXTERNAL - ERTS_MONOTONIC_BEGIN)
#define ERTS_MONOTONIC_OFFSET_NSEC					\
    ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
#define ERTS_MONOTONIC_OFFSET_USEC					\
    ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
#define ERTS_MONOTONIC_OFFSET_MSEC					\
    ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
#define ERTS_MONOTONIC_OFFSET_SEC					\
    ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE)

#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
    ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))
#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
    ((TCKS) * (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))

#else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
# 296 "beam/erl_time.h"

/*
 * Initialized in erts_init_sys_time_sup()
 */
#define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit)

/*
 * Offsets from internal monotonic time to external monotonic time
 *
 * Initialized in erts_init_time_sup()...
 */
#define ERTS_MONOTONIC_TIME_START_EXTERNAL (erts_time_sup__.r.o.start)
#define ERTS_MONOTONIC_OFFSET_NATIVE (erts_time_sup__.r.o.start_offset.native)
#define ERTS_MONOTONIC_OFFSET_NSEC (erts_time_sup__.r.o.start_offset.nsec)
#define ERTS_MONOTONIC_OFFSET_USEC (erts_time_sup__.r.o.start_offset.usec)
#define ERTS_MONOTONIC_OFFSET_MSEC (erts_time_sup__.r.o.start_offset.msec)
#define ERTS_MONOTONIC_OFFSET_SEC (erts_time_sup__.r.o.start_offset.sec)

#define ERTS_CONV_FROM_MON_UNIT___(M, TO)				\
    ((ErtsMonotonicTime)						\
     erts_time_unit_conversion((Uint64) (M),				\
			       (Uint32) ERTS_MONOTONIC_TIME_UNIT,	\
			       (Uint32) (TO)))

#define ERTS_CONV_TO_MON_UNIT___(M, FROM)				\
    ((ErtsMonotonicTime)						\
     erts_time_unit_conversion((Uint64) (M),				\
			       (Uint32) (FROM),				\
			       (Uint32) ERTS_MONOTONIC_TIME_UNIT))	\

#define ERTS_MONOTONIC_TO_SEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1)
#define ERTS_MONOTONIC_TO_MSEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1000)
#define ERTS_MONOTONIC_TO_USEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000)
#define ERTS_MONOTONIC_TO_NSEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000*1000)

#define ERTS_SEC_TO_MONOTONIC__(SEC) \
    ERTS_CONV_TO_MON_UNIT___((SEC), 1)
#define ERTS_MSEC_TO_MONOTONIC__(MSEC) \
    ERTS_CONV_TO_MON_UNIT___((MSEC), 1000)
#define ERTS_USEC_TO_MONOTONIC__(USEC) \
    ERTS_CONV_TO_MON_UNIT___((USEC), 1000*1000)
#define ERTS_NSEC_TO_MONOTONIC__(NSEC) \
    ERTS_CONV_TO_MON_UNIT___((NSEC), 1000*1000*1000)

#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
    ERTS_CONV_FROM_MON_UNIT___((MON), ERTS_CLKTCK_RESOLUTION)
#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
    ERTS_CONV_TO_MON_UNIT___((TCKS), ERTS_CLKTCK_RESOLUTION)

#endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
# 350 "beam/erl_time.h"

#define ERTS_MONOTONIC_TIME_END_EXTERNAL				\
    (ERTS_MONOTONIC_TIME_START_EXTERNAL < 0				\
     ? (ERTS_MONOTONIC_TIME_START_EXTERNAL				\
	+ (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN))			\
     : (ERTS_MONOTONIC_END - ERTS_MONOTONIC_TIME_START_EXTERNAL))

#define ERTS_MSEC_TO_CLKTCKS__(MON) \
    ((MON) * (ERTS_CLKTCK_RESOLUTION/1000))
#define ERTS_CLKTCKS_TO_MSEC__(TCKS) \
    ((TCKS) / (ERTS_CLKTCK_RESOLUTION/1000))

#define ERTS_MONOTONIC_TO_SEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_SEC__((X)))
#define ERTS_MONOTONIC_TO_MSEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_MSEC__((X)))
#define ERTS_MONOTONIC_TO_USEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_USEC__((X)))
#define ERTS_MONOTONIC_TO_NSEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_NSEC__((X)))
#define ERTS_SEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_SEC_TO_MONOTONIC__((X)))
#define ERTS_MSEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MSEC_TO_MONOTONIC__((X)))
#define ERTS_USEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_USEC_TO_MONOTONIC__((X)))
#define ERTS_NSEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_NSEC_TO_MONOTONIC__((X)))

#define ERTS_MONOTONIC_TO_CLKTCKS(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_MONOTONIC_TO_CLKTCKS__((X)))
#define ERTS_CLKTCKS_TO_MONOTONIC(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_CLKTCKS_TO_MONOTONIC__((X)))

#define ERTS_MSEC_TO_CLKTCKS(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_MSEC_TO_CLKTCKS__((X)))
#define ERTS_CLKTCKS_TO_MSEC(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_CLKTCKS_TO_MSEC__((X)))

#endif /* ERL_TIME_H__ */
# 402 "beam/erl_time.h"

/* timer-wheel api */
#if defined(ERTS_WANT_TIMER_WHEEL_API) && !defined(ERTS_GOT_TIMER_WHEEL_API)
#define ERTS_GOT_TIMER_WHEEL_API

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 407 "beam/erl_time.h"
# 408 "beam/erl_time.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 408 "beam/erl_time.h"
# 409 "beam/erl_time.h"

void erts_sched_init_time_sup(ErtsSchedulerData *esdp);


#define ERTS_TWHEEL_SLOT_AT_ONCE -1
#define ERTS_TWHEEL_SLOT_INACTIVE -2

/*
** Timer entry:
*/
typedef struct erl_timer {
    struct erl_timer* next;	/* next entry tiw slot or chain */
    struct erl_timer* prev;	/* prev entry tiw slot or chain */
    union {
	struct {
	    void (*timeout)(void*); /* called when timeout */
	    void (*cancel)(void*);  /* called when cancel (may be NULL) */
	    void* arg;              /* argument to timeout/cancel procs */
	} func;
	ErtsThrPrgrLaterOp cleanup;
    } u;
    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
    int slot;
} ErtsTWheelTimer;

typedef void (*ErlTimeoutProc)(void*);
typedef void (*ErlCancelProc)(void*);

void erts_twheel_set_timer(ErtsTimerWheel *tiw,
			   ErtsTWheelTimer *p, ErlTimeoutProc timeout,
			   ErlCancelProc cancel, void *arg,
			   ErtsMonotonicTime timeout_pos);
void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p);
ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp);

ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *);

ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p);
ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p)
{
    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
}

ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
{
    return *((ErtsMonotonicTime *) nxt_tmo_ref);
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 462 "beam/erl_time.h"

void
erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
			  void (*tclbk)(void *),
			  void (*func)(void *,
				       ErtsMonotonicTime,
				       void *),
			  void *arg);

#endif /* timer wheel api */
# 472 "beam/erl_time.h"
# 33 "beam/erl_ptab.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_utils.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_ptab.h"
# 1 "beam/erl_utils.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_UTILS_H__
#define ERL_UTILS_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_utils.h"
# 25 "beam/erl_utils.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/erl_utils.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 29 "beam/erl_smp.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 26 "beam/erl_utils.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_printf.h"
#endif /* expanded by -frewrite-includes */
# 26 "beam/erl_utils.h"
# 27 "beam/erl_utils.h"

struct process;

typedef struct {
#ifdef DEBUG
    int smp_api;
#endif
# 34 "beam/erl_utils.h"
    union {
	Uint64 not_atomic;
	erts_atomic64_t atomic;
    } counter;
} erts_interval_t;

void erts_interval_init(erts_interval_t *);
void erts_smp_interval_init(erts_interval_t *);
Uint64 erts_step_interval_nob(erts_interval_t *);
Uint64 erts_step_interval_relb(erts_interval_t *);
Uint64 erts_smp_step_interval_nob(erts_interval_t *);
Uint64 erts_smp_step_interval_relb(erts_interval_t *);
Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
ERTS_GLB_INLINE Uint64 erts_current_interval_acqb(erts_interval_t *);
ERTS_GLB_INLINE Uint64 erts_smp_current_interval_nob(erts_interval_t *);
ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Uint64
erts_current_interval_nob__(erts_interval_t *icp)
{
    return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
}

ERTS_GLB_INLINE Uint64
erts_current_interval_acqb__(erts_interval_t *icp)
{
    return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
}

ERTS_GLB_INLINE Uint64
erts_current_interval_nob(erts_interval_t *icp)
{
    ASSERT(!icp->smp_api);
    return erts_current_interval_nob__(icp);
}

ERTS_GLB_INLINE Uint64
erts_current_interval_acqb(erts_interval_t *icp)
{
    ASSERT(!icp->smp_api);
    return erts_current_interval_acqb__(icp);
}

ERTS_GLB_INLINE Uint64
erts_smp_current_interval_nob(erts_interval_t *icp)
{
    ASSERT(icp->smp_api);
#ifdef ERTS_SMP
    return erts_current_interval_nob__(icp);
#else
# 92 "beam/erl_utils.h"
    return icp->counter.not_atomic;
#endif
# 94 "beam/erl_utils.h"
}

ERTS_GLB_INLINE Uint64
erts_smp_current_interval_acqb(erts_interval_t *icp)
{
    ASSERT(icp->smp_api);
#ifdef ERTS_SMP
    return erts_current_interval_acqb__(icp);
#else
# 103 "beam/erl_utils.h"
    return icp->counter.not_atomic;
#endif
# 105 "beam/erl_utils.h"
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 108 "beam/erl_utils.h"

/*
 * To be used to silence unused result warnings, but do not abuse it.
 */
void erts_silence_warn_unused_result(long unused);


int erts_fit_in_bits_int64(Sint64);
int erts_fit_in_bits_int32(Sint32);
int erts_fit_in_bits_uint(Uint);
int erts_list_length(Eterm);
int erts_is_builtin(Eterm, Eterm, int);
Uint32 make_broken_hash(Eterm);
Uint32 block_hash(byte *, unsigned, Uint32);
Uint32 make_hash2(Eterm);
Uint32 make_hash(Eterm);
Uint32 make_internal_hash(Eterm);

void erts_save_emu_args(int argc, char **argv);
Eterm erts_get_emu_args(struct process *c_p);
Eterm erts_get_ethread_info(struct process * c_p);

Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str);
Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui);
Eterm erts_bld_uword(Uint **hpp, Uint *szp, UWord uw);
Eterm erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64);
Eterm erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64);
Eterm erts_bld_cons(Uint **hpp, Uint *szp, Eterm car, Eterm cdr);
Eterm erts_bld_tuple(Uint **hpp, Uint *szp, Uint arity, ...);
#define erts_bld_tuple2(H,S,E1,E2) erts_bld_tuple(H,S,2,E1,E2)
#define erts_bld_tuple3(H,S,E1,E2,E3) erts_bld_tuple(H,S,3,E1,E2,E3)
#define erts_bld_tuple4(H,S,E1,E2,E3,E4) erts_bld_tuple(H,S,4,E1,E2,E3,E4)
#define erts_bld_tuple5(H,S,E1,E2,E3,E4,E5) erts_bld_tuple(H,S,5,E1,E2,E3,E4,E5)
Eterm erts_bld_tuplev(Uint **hpp, Uint *szp, Uint arity, Eterm terms[]);
Eterm erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len);
#define erts_bld_string(hpp,szp,str) erts_bld_string_n(hpp,szp,str,strlen(str))
Eterm erts_bld_list(Uint **hpp, Uint *szp, Sint length, Eterm terms[]);
Eterm erts_bld_2tup_list(Uint **hpp, Uint *szp,
			 Sint length, Eterm terms1[], Uint terms2[]);
Eterm
erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp,
			     Sint length, Eterm atoms[], UWord uints[]);
Eterm
erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length,
			      Eterm atoms[], Uint uints1[], Uint uints2[]);

void erts_init_utils(void);
void erts_init_utils_mem(void);

erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint);
void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *);

#if HALFWORD_HEAP
int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base);
#  define eq(A,B) eq_rel(A,NULL,B,NULL)
#else
# 164 "beam/erl_utils.h"
int eq(Eterm, Eterm);
#  define eq_rel(A,A_BASE,B,B_BASE) eq(A,B)
#endif
# 167 "beam/erl_utils.h"

#define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y))))

#if HALFWORD_HEAP
Sint erts_cmp_rel_opt(Eterm, Eterm*, Eterm, Eterm*, int, int);
#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp_rel_opt(A,A_BASE,B,B_BASE,0,0)
#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp_rel_opt(A,A_BASE,B,B_BASE,1,0)
#define CMP(A,B)                         erts_cmp_rel_opt(A,NULL,B,NULL,0,0)
#define CMP_TERM(A,B)                    erts_cmp_rel_opt(A,NULL,B,NULL,1,0)
#define CMP_EQ_ONLY(A,B)                 erts_cmp_rel_opt(A,NULL,B,NULL,0,1)
#else
# 178 "beam/erl_utils.h"
Sint erts_cmp(Eterm, Eterm, int, int);
Sint cmp(Eterm a, Eterm b);
#define cmp_rel(A,A_BASE,B,B_BASE)       erts_cmp(A,B,0,0)
#define cmp_rel_term(A,A_BASE,B,B_BASE)  erts_cmp(A,B,1,0)
#define CMP(A,B)                         erts_cmp(A,B,0,0)
#define CMP_TERM(A,B)                    erts_cmp(A,B,1,0)
#define CMP_EQ_ONLY(A,B)                 erts_cmp(A,B,0,1)
#endif
# 186 "beam/erl_utils.h"

#define cmp_lt(a,b)          (CMP((a),(b)) <  0)
#define cmp_le(a,b)          (CMP((a),(b)) <= 0)
#define cmp_eq(a,b)          (CMP_EQ_ONLY((a),(b)) == 0)
#define cmp_ne(a,b)          (CMP_EQ_ONLY((a),(b)) != 0)
#define cmp_ge(a,b)          (CMP((a),(b)) >= 0)
#define cmp_gt(a,b)          (CMP((a),(b)) >  0)

#define cmp_lt_term(a,b)     (CMP_TERM((a),(b)) <  0)
#define cmp_le_term(a,b)     (CMP_TERM((a),(b)) <= 0)
#define cmp_ge_term(a,b)     (CMP_TERM((a),(b)) >= 0)
#define cmp_gt_term(a,b)     (CMP_TERM((a),(b)) >  0)

#define CMP_LT(a,b)          ((a) != (b) && cmp_lt((a),(b)))
#define CMP_GE(a,b)          ((a) == (b) || cmp_ge((a),(b)))
#define CMP_EQ(a,b)          ((a) == (b) || cmp_eq((a),(b)))
#define CMP_NE(a,b)          ((a) != (b) && cmp_ne((a),(b)))

#define CMP_LT_TERM(a,b)     ((a) != (b) && cmp_lt_term((a),(b)))
#define CMP_GE_TERM(a,b)     ((a) == (b) || cmp_ge_term((a),(b)))

#endif
# 208 "beam/erl_utils.h"
# 34 "beam/erl_ptab.h" 2
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 35 "beam/erl_ptab.h"
# 1 "beam/erl_thr_progress.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread progress information. Used by lock free algorithms
 *              to determine when all involved threads are guaranteed to
 *              have passed a specific point of execution.
 *
 *              Usage instructions can be found in ert_thr_progress.c
 *
 * Author: 	Rickard Green
 */

#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
#define ERL_THR_PROGRESS_H__TSD_TYPE__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/erl_thr_progress.h"
# 35 "beam/erl_thr_progress.h"

#ifndef ERTS_SMP

#define erts_smp_thr_progress_block() ((void) 0)
#define erts_smp_thr_progress_unblock() ((void) 0)
#define erts_smp_thr_progress_is_blocking() 1

#else /* ERTS_SMP */
# 43 "beam/erl_thr_progress.h"

#define erts_smp_thr_progress_block erts_thr_progress_block
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking

void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);

typedef Uint64 ErtsThrPrgrVal;

#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */

typedef struct {
    ErtsThrPrgrVal next;
    ErtsThrPrgrVal current;
    int chk_next_ix;
    struct {
	int current;
	int waiting;
    } umrefc_ix;
} ErtsThrPrgrLeaderState;

typedef struct {
    int id;
    int is_managed;
    int is_blocking;
#ifdef ERTS_ENABLE_LOCK_CHECK
    int is_delaying; /* managed is always delaying */
#endif
# 73 "beam/erl_thr_progress.h"
    int is_temporary;

    /* --- Part below only for registered threads --- */

    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];

    /* --- Part below only for managed threads --- */

    int leader; /* Needs to be first in the managed threads part */
    int active;
    ErtsThrPrgrVal confirmed;
    ErtsThrPrgrLeaderState leader_state;
} ErtsThrPrgrData;

int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
void erts_thr_progress_fatal_error_wait(SWord timeout);

#endif /* ERTS_SMP */
# 91 "beam/erl_thr_progress.h"

typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
struct ErtsThrPrgrLaterOp_ {
#ifdef ERTS_SMP
    ErtsThrPrgrVal later;
#endif
# 97 "beam/erl_thr_progress.h"
    void (*func)(void *);
    void *data;
    ErtsThrPrgrLaterOp *next;
};

#endif
# 103 "beam/erl_thr_progress.h"

#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
#define ERL_THR_PROGRESS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 107 "beam/erl_thr_progress.h"
# 108 "beam/erl_thr_progress.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 108 "beam/erl_thr_progress.h"
# 109 "beam/erl_thr_progress.h"

#ifdef ERTS_SMP

/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))

extern erts_tsd_key_t erts_thr_prgr_data_key__;

#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t

typedef struct {
    void *arg;
    void (*wakeup)(void *);
    void (*prepare_wait)(void *);
    void (*wait)(void *);
    void (*finalize_wait)(void *);
} ErtsThrPrgrCallbacks;

typedef struct {
    ERTS_THR_PRGR_ATOMIC current;
} ErtsThrPrgr;

typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)

extern ErtsThrPrgr erts_thr_prgr__;

void erts_thr_progress_pre_init(void);
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
					       ErtsThrPrgrCallbacks *,
					       int);
void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
			      ErtsThrPrgrVal value);
int erts_thr_progress_update(ErtsSchedulerData *esdp);
int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
void erts_thr_progress_unmanaged_continue__(int umrefc_ix);

void erts_thr_progress_dbg_print_state(void);

ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);

ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
#endif
# 168 "beam/erl_thr_progress.h"
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
					    ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
}

ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_managed;
}

ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)
{
    if (erts_thr_progress_is_managed_thread())
	return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
    else
	return erts_thr_progress_unmanaged_delay__();
}

ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
{
    ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
	   || erts_thr_progress_is_managed_thread());
    if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
	erts_thr_progress_unmanaged_continue__(handle);
}

#ifdef ERTS_ENABLE_LOCK_CHECK

ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_delaying;
}

#endif
# 233 "beam/erl_thr_progress.h"

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
{
    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
	return ((ErtsThrPrgrVal) 0);
    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
	return ((ErtsThrPrgrVal) 1);
    else
	return val + ((ErtsThrPrgrVal) 2);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData *esdp)
{
    ErtsThrPrgrData *tpd;
    ErtsThrPrgrVal val;
    if (esdp) {
	tpd = &esdp->thr_progress_data;
    managed_thread:
	val = tpd->confirmed;
	ERTS_THR_MEMORY_BARRIER;
    }
    else {
	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
	if (tpd && tpd->is_managed)
	    goto managed_thread;
	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
    }
    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
    return erts_thr_progress_current_to_later__(val);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
    else
	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}

ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
{
    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
	/* May have wrapped... */
	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
	    /*
	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
	     * these ranges 'current' is considered later than 'val0'.
	     */
	    return 1;
	}
    }
    return val1 > val0;
}

ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
{
    if (this == val)
	return 1;
    return erts_thr_progress_has_passed__(this, val);
}

ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
}

ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    if (val1 == val2)
	return 0;
    if (erts_thr_progress_has_passed__(val1, val2))
	return 1;
    else
	return -1;
}	

ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)
{
    ErtsThrPrgrVal current = erts_thr_progress_current();
    return erts_thr_progress_has_reached_this(current, val);
}

#endif
# 326 "beam/erl_thr_progress.h"

#endif /* ERTS_SMP */
# 328 "beam/erl_thr_progress.h"

#endif
# 330 "beam/erl_thr_progress.h"
# 36 "beam/erl_ptab.h" 2
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc.h"
#endif /* expanded by -frewrite-includes */
# 37 "beam/erl_ptab.h"
# 38 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 38 "beam/erl_ptab.h"
# 1 "beam/erl_monitors.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2004-2013. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/**********************************************************************
 * Header for monitors and links data structures.
 * Monitors are kept in an AVL tree and the data structures for
 * the four different types of monitors are like this:
 **********************************************************************
 * Local monitor by pid/port: 
 * (Ref is always same in all involved data structures)
 **********************************************************************
 *        Process/Port X     Process Y
 *       +-------------+    +-------------+
 * Type: | MON_ORIGIN  |    | MON_TARGET  |
 *       +-------------+    +-------------+
 * Pid:  | Pid(Y)      |    | Pid/Port(X) |
 *       +-------------+    +-------------+
 * Name: | []          |    | []          |
 *       +-------------+    +-------------+
 **********************************************************************
 * Local monitor by name: (Ref is always same in all involved data structures)
 **********************************************************************
 *        Process X          Process Y (name foo)
 *       +-------------+    +-------------+
 * Type: | MON_ORIGIN  |    | MON_TARGET  |
 *       +-------------+    +-------------+
 * Pid:  | Pid(Y)      |    | Pid(X)      |
 *       +-------------+    +-------------+
 * Name: | Atom(foo)   |    | Atom(foo)   |
 *       +-------------+    +-------------+
 **********************************************************************
 * Remote monitor by pid: (Ref is always same in all involved data structures)
 **********************************************************************
 *                    Node A              |           Node B
 *       ---------------------------------+----------------------------------
 *        Process X (@A)   Distentry @A       Distentry @B     Process Y (@B)
 *                         for node B         for node A 
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * Type: | MON_ORIGIN  |  | MON_TARGET  |    | MON_ORIGIN  |  | MON_TARGET  |
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * Pid:  | Pid(Y)      |  | Pid(X)      |    | Pid(Y)      |  | Pid(X)      |
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * Name: | []          |  | []          |    | []          |  | []          |
 *       +-------------+  +-------------+    +-------------+  +-------------+
 **********************************************************************
 * Remote monitor by name: (Ref is always same in all involved data structures)
 **********************************************************************
 *                    Node A              |           Node B
 *       ---------------------------------+----------------------------------
 *        Process X (@A)   Distentry @A       Distentry @B     Process Y (@B)
 *                         for node B         for node A       (name foo)
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * Type: | MON_ORIGIN  |  | MON_TARGET  |    | MON_ORIGIN  |  | MON_TARGET  |
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * Pid:  | Atom(node B)|  | Pid(X)      |    | Pid(Y)      |  | Pid(X)      |
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * Name: | Atom(foo)   |  | Atom(foo)   |    | Atom(foo)   |  | Atom(foo)   |
 *       +-------------+  +-------------+    +-------------+  +-------------+
 * The reason for the node atom in X->pid is that we don't know the actual
 * pid of the monitored process on the other node when setting the monitor
 * (which is done asyncronously).
 **********************************************************************/
#ifndef _ERL_MONITORS_H
#define _ERL_MONITORS_H

/* Type tags for monitors */
#define MON_ORIGIN 1
#define MON_TARGET 3
#define MON_TIME_OFFSET 7

/* Type tags for links */
#define LINK_PID 1   /* ...Or port */
#define LINK_NODE 3  /* "Node monitor" */  

/* Size of a monitor without heap, in words (fixalloc) */
#define ERTS_MONITOR_SIZE ((sizeof(ErtsMonitor) - sizeof(Uint))/sizeof(Uint))
#define ERTS_MONITOR_SH_SIZE (ERTS_MONITOR_SIZE + REF_THING_SIZE)
#define ERTS_LINK_SIZE ((sizeof(ErtsLink) - sizeof(Uint))/sizeof(Uint))
#define ERTS_LINK_SH_SIZE ERTS_LINK_SIZE /* Size of fix-alloced links */

/* ErtsMonitor and ErtsLink *need* to begin in a similar way as 
   ErtsMonitorOrLink */
typedef struct erts_monitor_or_link {
    struct erts_monitor_or_link *left, *right;
    Sint16 balance;    
} ErtsMonitorOrLink;

typedef struct erts_monitor {
    struct erts_monitor *left, *right; 
    Sint16 balance;
    Uint16 type;  /* MON_ORIGIN | MON_TARGET | MON_TIME_OFFSET */
    Eterm ref;
    Eterm pid;    /* In case of distributed named monitor, this is the
		     nodename atom in MON_ORIGIN process, otherwise a pid or
		     , in case of a MON_TARGET, a port */
    Eterm name;   /* When monitoring a named process: atom() else [] */
    Uint heap[1]; /* Larger in reality */
} ErtsMonitor;

typedef struct erts_link {
    struct erts_link *left, *right;
    Sint16 balance;
    Uint16 type;             /* LINK_PID | LINK_NODE */
    Eterm pid;               /* When node monitor, 
				the node atom is here instead */
    union {
	struct erts_link *root;  /* Used only in dist entries */
	Uint refc;
    } shared;
    Uint heap[1];            /* Larger in reality */
} ErtsLink;   

typedef struct erts_suspend_monitor {
    struct erts_suspend_monitor *left, *right;
    Sint16 balance;

    int pending;
    int active;
    Eterm pid;
} ErtsSuspendMonitor;

#define ERTS_LINK_ROOT(Linkp) ((Linkp)->shared.root)
#define ERTS_LINK_REFC(Linkp) ((Linkp)->shared.refc) 

Uint erts_tot_link_lh_size(void);


/* Prototypes */
void erts_destroy_monitor(ErtsMonitor *mon);
void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid,
		      Eterm name);
ErtsMonitor *erts_remove_monitor(ErtsMonitor **root, Eterm ref);
ErtsMonitor *erts_lookup_monitor(ErtsMonitor *root, Eterm ref);
void erts_sweep_monitors(ErtsMonitor *root, 
			 void (*doit)(ErtsMonitor *, void *),
			 void *context);

void erts_destroy_link(ErtsLink *lnk);
/* Returns 0 if OK, < 0 if already present */
int erts_add_link(ErtsLink **root, Uint type, Eterm pid);
ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid);
ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid);
ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid);
void erts_sweep_links(ErtsLink *root, 
		      void (*doit)(ErtsLink *, void *),
		      void *context);

void erts_destroy_suspend_monitor(ErtsSuspendMonitor *sproc);
void erts_sweep_suspend_monitors(ErtsSuspendMonitor *root,
				 void (*doit)(ErtsSuspendMonitor *, void *),
				 void *context);
ErtsSuspendMonitor *erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root,
						       Eterm pid);
ErtsSuspendMonitor *erts_lookup_suspend_monitor(ErtsSuspendMonitor *root,
						Eterm pid);
void erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid);
void erts_init_monitors(void);
void erts_one_link_size(ErtsLink *lnk, void *vpu);
void erts_one_mon_size(ErtsMonitor *mon, void *vpu);

#define erts_doforall_monitors erts_sweep_monitors
#define erts_doforall_links erts_sweep_links
#define erts_doforall_suspend_monitors erts_sweep_suspend_monitors

#endif /* _ERL_MONITORS_H */
# 183 "beam/erl_monitors.h"
# 39 "beam/erl_ptab.h" 2

#define ERTS_TRACER_PROC(P) 	((P)->common.tracer_proc)
#define ERTS_TRACE_FLAGS(P)	((P)->common.trace_flags)

#define ERTS_P_LINKS(P)		((P)->common.u.alive.links)
#define ERTS_P_MONITORS(P)	((P)->common.u.alive.monitors)

#define IS_TRACED(p) \
    (ERTS_TRACER_PROC((p)) != NIL)
#define ARE_TRACE_FLAGS_ON(p,tf) \
    ((ERTS_TRACE_FLAGS((p)) & (tf|F_SENSITIVE)) == (tf))
#define IS_TRACED_FL(p,tf) \
    ( IS_TRACED(p) && ARE_TRACE_FLAGS_ON(p,tf) )

typedef struct {
    Eterm id;
    union {
	erts_atomic_t atmc;
	Sint sint;
    } refc;
    Eterm tracer_proc;
    Uint trace_flags;
    erts_smp_atomic_t timer;
    union {
	/* --- While being alive --- */
	struct {
	    Uint64 started_interval;
	    struct reg_proc *reg;
	    ErtsLink *links;
	    ErtsMonitor *monitors;
	} alive;

	/* --- While being released --- */
	ErtsThrPrgrLaterOp release;
    } u;
} ErtsPTabElementCommon;

typedef struct ErtsPTabDeletedElement_ ErtsPTabDeletedElement;

typedef struct {
    erts_smp_rwmtx_t rwmtx;
    erts_interval_t interval;
    struct {
	ErtsPTabDeletedElement *start;
	ErtsPTabDeletedElement *end;
    } deleted;
    int chunks;
} ErtsPTabListData;

typedef struct {
    erts_smp_atomic64_t last_data;
    erts_smp_atomic32_t count;
    erts_smp_atomic32_t aid_ix;
    erts_smp_atomic32_t fid_ix;
} ErtsPTabVolatileData;

typedef struct {
    erts_smp_atomic_t *tab;
    erts_smp_atomic32_t *free_id_data;
    Uint32 max;
    Uint32 pix_mask;
    Uint32 pix_cl_mask;
    Uint32 pix_cl_shift;
    Uint32 pix_cli_mask;
    Uint32 pix_cli_shift;
    Uint32 dix_cl_mask;
    Uint32 dix_cl_shift;
    Uint32 dix_cli_mask;
    Uint32 dix_cli_shift;
    ErtsPTabElementCommon *invalid_element;
    Eterm invalid_data;
    void (*release_element)(void *);
    UWord element_size;
    int atomic_refc;
} ErtsPTabReadOnlyData;

typedef struct {
    /*
     * Data mainly modified when someone is listing
     * the content of the table.
     */
    union {
	ErtsPTabListData data;
	char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabListData))];
    } list;

    /*
     * Frequently modified data.
     */
    union {
	ErtsPTabVolatileData tile;
	char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabVolatileData))];
    } vola;

    /*
     * Read only data.
     */
    union {
	ErtsPTabReadOnlyData o;
	char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabReadOnlyData))];
    } r;
} ErtsPTab;

#define ERTS_PTAB_ID_DATA_SIZE	28
#define ERTS_PTAB_ID_DATA_SHIFT	(_TAG_IMMED1_SIZE)
/* ERTS_PTAB_MAX_SIZE must be a power of 2 */
#define ERTS_PTAB_MAX_SIZE (SWORD_CONSTANT(1) << 27)
#if (ERTS_PTAB_MAX_SIZE-1) > MAX_SMALL
# error "The maximum number of processes/ports must fit in a SMALL."
#endif
# 149 "beam/erl_ptab.h"


/*
 * Currently pids and ports are allowed.
 */
#if _PID_DATA_SIZE != ERTS_PTAB_ID_DATA_SIZE
# error "Unexpected pid data size"
#endif
# 157 "beam/erl_ptab.h"
#if _PID_DATA_SHIFT != ERTS_PTAB_ID_DATA_SHIFT
# error "Unexpected pid tag size"
#endif
# 160 "beam/erl_ptab.h"
#if _PORT_DATA_SIZE != ERTS_PTAB_ID_DATA_SIZE
# error "Unexpected port data size"
#endif
# 163 "beam/erl_ptab.h"
#if _PORT_DATA_SHIFT != ERTS_PTAB_ID_DATA_SHIFT
# error "Unexpected port tag size"
#endif
# 166 "beam/erl_ptab.h"

#define ERTS_PTAB_INVALID_ID(TAG)					\
    ((Eterm)								\
     ((((1 << ERTS_PTAB_ID_DATA_SIZE) - 1) << ERTS_PTAB_ID_DATA_SHIFT)	\
      | (TAG)))

#define erts_ptab_is_valid_id(ID)					\
    (is_internal_pid((ID)) || is_internal_port((ID)))

void erts_ptab_init(void);
void erts_ptab_init_table(ErtsPTab *ptab,
			  ErtsAlcType_t atype,
			  void (*release_element)(void *),
			  ErtsPTabElementCommon *invalid_element,
			  int size,
			  UWord element_size,
			  char *name,
			  int legacy,
			  int atomic_refc);
int erts_ptab_new_element(ErtsPTab *ptab,
			  ErtsPTabElementCommon *ptab_el,
			  void *init_arg,
			  void (*init_ptab_el)(void *, Eterm));
void erts_ptab_delete_element(ErtsPTab *ptab,
			      ErtsPTabElementCommon *ptab_el);
int erts_ptab_initialized(ErtsPTab *ptab);
UWord erts_ptab_mem_size(ErtsPTab *ptab);

ERTS_GLB_INLINE erts_interval_t *erts_ptab_interval(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_count(ErtsPTab *ptab);
ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata);
ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata);
ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data);
ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data);
ERTS_GLB_INLINE Eterm erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag);
ERTS_GLB_INLINE int erts_ptab_id2pix(ErtsPTab *ptab, Eterm id);
ERTS_GLB_INLINE Uint erts_ptab_id2data(ErtsPTab *ptab, Eterm id);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
					     Sint add_refc);
ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el,
						  Sint add_refc);
ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab);
ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab);
ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab);
ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_interval_t *
erts_ptab_interval(ErtsPTab *ptab)
{
    return &ptab->list.data.interval;
}

ERTS_GLB_INLINE int
erts_ptab_max(ErtsPTab *ptab)
{
    int max = ptab->r.o.max;
    return max == ERTS_PTAB_MAX_SIZE ? max - 1 : max;
}

ERTS_GLB_INLINE int
erts_ptab_count(ErtsPTab *ptab)
{
    int max = ptab->r.o.max;
    erts_aint32_t res = erts_smp_atomic32_read_nob(&ptab->vola.tile.count);
    if (max == ERTS_PTAB_MAX_SIZE) {
	max--;
	res--;
    }
    if (res > max)
	return max;
    ASSERT(res >= 0);
    return (int) res;

}

ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata)
{
    Uint32 data = ((Uint32) pixdata) & ~ptab->r.o.pix_mask;
    data |= (pixdata >> ptab->r.o.pix_cl_shift) & ptab->r.o.pix_cl_mask;
    data |= (pixdata & ptab->r.o.pix_cli_mask) << ptab->r.o.pix_cli_shift;
    return data;
}

ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata)
{
    return ((Uint32) pixdata) & ptab->r.o.pix_mask;
}

ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data)
{
    Uint32 n, pix;
    n = (Uint32) data;
    pix = ((n & ptab->r.o.pix_cl_mask) << ptab->r.o.pix_cl_shift);
    pix += ((n >> ptab->r.o.pix_cli_shift) & ptab->r.o.pix_cli_mask);
    ASSERT(0 <= pix && pix < ptab->r.o.max);
    return pix;
}

ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data)
{
    Uint pixdata = data & ~((Uint) ptab->r.o.pix_mask);
    pixdata |= (Uint) erts_ptab_data2pix(ptab, data);
    ASSERT(data == erts_ptab_pixdata2data(ptab, pixdata));
    return pixdata;
}

#if ERTS_SIZEOF_TERM == 8

ERTS_GLB_INLINE Eterm
erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag)
{
    HUint huint;
    Uint32 low_data = (Uint32) data;
    low_data &= (1 << ERTS_PTAB_ID_DATA_SIZE) - 1;
    low_data <<= ERTS_PTAB_ID_DATA_SHIFT;
    huint.hval[ERTS_HUINT_HVAL_HIGH] = erts_ptab_data2pix(ptab, data);
    huint.hval[ERTS_HUINT_HVAL_LOW] = low_data | ((Uint32) tag);
    return (Eterm) huint.val;
}

ERTS_GLB_INLINE int
erts_ptab_id2pix(ErtsPTab *ptab, Eterm id)
{
    HUint huint;
    huint.val = id;
    return (int) huint.hval[ERTS_HUINT_HVAL_HIGH];
}

ERTS_GLB_INLINE Uint
erts_ptab_id2data(ErtsPTab *ptab, Eterm id)
{
    HUint huint;
    huint.val = id;
    return (Uint) (huint.hval[ERTS_HUINT_HVAL_LOW] >> ERTS_PTAB_ID_DATA_SHIFT);
}

#elif ERTS_SIZEOF_TERM == 4
# 320 "beam/erl_ptab.h"

ERTS_GLB_INLINE Eterm
erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag)
{
    Eterm id;
    data &= ((1 << ERTS_PTAB_ID_DATA_SIZE) - 1);
    id = (Eterm) erts_ptab_data2pixdata(ptab, data);
    return (id << ERTS_PTAB_ID_DATA_SHIFT) | tag;
}

ERTS_GLB_INLINE int
erts_ptab_id2pix(ErtsPTab *ptab, Eterm id)
{
    Uint pixdata = (Uint) id;
    pixdata >>= ERTS_PTAB_ID_DATA_SHIFT;
    return (int) erts_ptab_pixdata2pix(ptab, pixdata);
}

ERTS_GLB_INLINE Uint
erts_ptab_id2data(ErtsPTab *ptab, Eterm id)
{
    Uint pixdata = (Uint) id;
    pixdata >>= ERTS_PTAB_ID_DATA_SHIFT;
    return erts_ptab_pixdata2data(ptab, pixdata);
}

#else
# 347 "beam/erl_ptab.h"
#error "Unsupported size of term"
#endif
# 349 "beam/erl_ptab.h"

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_ddrb(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_rb(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_aint_t refc = erts_atomic_inc_read_nob(&ptab_el->refc.atmc);
    ERTS_LC_ASSERT(refc > 1);
#else
# 380 "beam/erl_ptab.h"
    erts_atomic_inc_nob(&ptab_el->refc.atmc);
#endif
# 382 "beam/erl_ptab.h"
}

ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el)
{
    erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc);
    ERTS_SMP_LC_ASSERT(refc >= 0);
#ifdef ERTS_SMP
    if (refc == 0)
	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif
# 392 "beam/erl_ptab.h"
    return (Sint) refc;
}

ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el,
						  Sint add_refc)
{
    erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc,
					       (erts_aint_t) add_refc);
    ERTS_SMP_LC_ASSERT(refc >= 0);
    return (Sint) refc;
}

ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el)
{
    return (Sint) erts_atomic_read_nob(&ptab_el->refc.atmc);
}

ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el)
{
    ptab_el->refc.sint++;
    ASSERT(ptab_el->refc.sint > 1);
}

ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el)
{
    Sint refc = --ptab_el->refc.sint;
    ERTS_SMP_LC_ASSERT(refc >= 0);
    return refc;
}

ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
					     Sint add_refc)
{
    ptab_el->refc.sint += add_refc;
    ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0);
    return (Sint) ptab_el->refc.sint;
}

ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el)
{
    return ptab_el->refc.sint;
}

ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_rlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab)
{
    return erts_smp_rwmtx_tryrlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_runlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_rwlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab)
{
    return erts_smp_rwmtx_tryrwlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_rwunlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab)
{
    return erts_smp_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab)
{
    return erts_smp_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx);
}

#endif
# 476 "beam/erl_ptab.h"

#endif
# 478 "beam/erl_ptab.h"

#if defined(ERTS_PTAB_WANT_BIF_IMPL__) && !defined(ERTS_PTAB_LIST__)
#define ERTS_PTAB_LIST__

#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 482 "beam/erl_ptab.h"
# 483 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "bif.h"
#endif /* expanded by -frewrite-includes */
# 483 "beam/erl_ptab.h"
# 484 "beam/erl_ptab.h"

BIF_RETTYPE erts_ptab_list(struct process *c_p, ErtsPTab *ptab);

#endif
# 488 "beam/erl_ptab.h"

#if defined(ERTS_PTAB_WANT_DEBUG_FUNCS__) && !defined(ERTS_PTAB_DEBUG_FUNCS__)
#define ERTS_PTAB_DEBUG_FUNCS__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/erl_ptab.h"
# 492 "beam/erl_ptab.h"

/* Debug functions */
Sint erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next);
Eterm erts_debug_ptab_list(Process *c_p, ErtsPTab *ptab);
Eterm erts_debug_ptab_list_bif_info(Process *c_p, ErtsPTab *ptab);

#endif
# 499 "beam/erl_ptab.h"
# 25 "beam/erl_node_container_utils.h" 2

/*
 * Note regarding node containers:
 *
 * The term "node container" is used as a group name (internally in
 * the emulator) for the Erlang data types that contain a reference
 * to a node, i.e. pids, ports, and references.
 *
 * Node containers are divided into internal and external node containers.
 * An internal node container refer to the current incarnation of the
 * node which it reside on. An external node container refer to
 * either a remote node (i.e. a node with another node name than the 
 * node name of the node on which the node container resides on) or another
 * incarnation of the node which the node container resides on (i.e
 * another node with the same node name but another creation).
 *
 * External node containers are boxed data types. The data of an
 * external node container is stored on the heap together with a pointer
 * to an element in the node table (see erl_term.h and erl_node_tables.h).
 * The elements of the node table are garbage collected by reference
 * counting (much like refc binaries, and funs in the separate heap case).
 *
 * Internal node containers are stored as they previously were (in R8)
 * with the exception of changed internal layouts (see erl_term.h), i.e.
 * internal pid, and internal port are immediate data types and internal
 * reference is a boxed data type. An internal node container have an
 * implicit reference to the 'erts_this_node' element in the node table.
 *
 * OBSERVE! Pids doesn't use fixed size 'serial' and 'number' fields any
 * more. Previously the 15 bit 'number' field of a pid was used as index
 * into the process table, and the 3 bit 'serial' field was used as a
 * "wrap counter". The needed number of bits for index into the process
 * table is now calculated at startup and the rest (of the 18 bits used)
 * are used as 'serial'. In the "emulator interface" (external format,
 * list_to_pid, etc) the least significant 15 bits are presented as
 * 'number' and the most significant 3 bits are presented as 'serial',
 * though. The makro internal_pid_index() can be used for retrieving
 * index into the process table. Do *not* use the result from
 * pid_number() as an index into the process table. The pid_number() and
 * pid_serial() (and friends) fetch the old fixed size 'number' and
 * 'serial' fields.
 */

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * Node containers                                                         *
\*                                                                         */

#define node_container_node_name(x)	(is_external(x)			\
					 ? external_node_name((x))	\
					 : internal_node_name((x)))
#define node_container_creation(x) 	(is_external(x)			\
					 ? external_creation((x))	\
					 : internal_creation((x)))
#define node_container_dist_entry(x) 	(is_external(x)			\
					 ? external_dist_entry((x))	\
					 : internal_dist_entry((x)))
#define node_container_channel_no(x)	(is_external((x))		\
					 ? external_channel_no((x))	\
					 : internal_channel_no((x)))
#define is_node_container(x)		(is_external((x)) || is_internal((x)))
#define is_not_node_container(x)	(!is_node_container((x)))

#define is_internal(x) 			(is_internal_pid((x))		\
					 || is_internal_port((x))	\
					 || is_internal_ref((x)))
#define is_not_internal(x) 		(!is_internal((x)))
#define internal_node_name(x)		(erts_this_node->sysname)
#define external_node_name(x)		external_node((x))->sysname
#define internal_creation(x)		(erts_this_node->creation)
#define external_creation(x)		(external_node((x))->creation)
#define internal_dist_entry(x)		(erts_this_node->dist_entry)
#define external_dist_entry(x)		(external_node((x))->dist_entry)

/*
 * For this node (and previous incarnations of this node), 0 is used as
 * channel no. For other nodes, the atom index of the atom corresponding
 * to the node name is used as channel no.
 *
 * (We used to assert for correct node names, but we removed that assertion
 *  as it is possible to sneak in incorrect node names for instance using
 *  the external format.)
 */
#define dist_entry_channel_no(x)				\
  ((x) == erts_this_dist_entry					\
   ? ((Uint) 0)							\
   : (ASSERT(is_atom((x)->sysname)),			        \
      (Uint) atom_val((x)->sysname)))
#define internal_channel_no(x) ((Uint) ERST_INTERNAL_CHANNEL_NO)
#define external_channel_no(x) \
  (dist_entry_channel_no(external_dist_entry((x))))

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * Pids                                                                    *
\*                                                                         */

extern ErtsPTab erts_proc;

#define make_internal_pid(D)		erts_ptab_make_id(&erts_proc, \
							  (D), \
							  _TAG_IMMED1_PID)

#define internal_pid_index(PID)		(ASSERT(is_internal_pid((PID))), \
					 erts_ptab_id2pix(&erts_proc, (PID)))

#define internal_pid_data(PID)		(ASSERT(is_internal_pid((PID))), \
					 erts_ptab_id2data(&erts_proc, (PID)))

#define internal_pid_number(x)		_GET_PID_NUM(internal_pid_data((x)))
#define internal_pid_serial(x)		_GET_PID_SER(internal_pid_data((x)))

#define internal_pid_node_name(x)	(internal_pid_node((x))->sysname)
#define external_pid_node_name(x)	(external_pid_node((x))->sysname)
#define internal_pid_creation(x)	(internal_pid_node((x))->creation)
#define external_pid_creation(x)	(external_pid_node((x))->creation)
#define internal_pid_dist_entry(x)	(internal_pid_node((x))->dist_entry)
#define external_pid_dist_entry(x)	(external_pid_node((x))->dist_entry)

#define internal_pid_channel_no(x)	(internal_channel_no((x)))
#define external_pid_channel_no(x)	(external_channel_no((x)))

#define pid_data_words(x)		(is_internal_pid((x))		\
					 ? internal_pid_data_words((x))	\
					 : external_pid_data_words((x)))
#define pid_number(x)			(is_internal_pid((x))		\
					 ? internal_pid_number((x))	\
					 : external_pid_number((x)))
#define pid_serial(x)			(is_internal_pid((x))		\
					 ? internal_pid_serial((x))	\
					 : external_pid_serial((x)))
#define pid_node(x)			(is_internal_pid((x))		\
					 ? internal_pid_node((x))	\
					 : external_pid_node((x)))
#define pid_node_name(x)		(is_internal_pid((x))		\
					 ? internal_pid_node_name((x))	\
					 : external_pid_node_name((x)))
#define pid_creation(x)			(is_internal_pid((x))		\
					 ? internal_pid_creation((x))	\
					 : external_pid_creation((x)))
#define pid_dist_entry(x)		(is_internal_pid((x))		\
					 ? internal_pid_dist_entry((x))	\
					 : external_pid_dist_entry((x)))
#define pid_channel_no(x)		(is_internal_pid((x))		\
					 ? internal_pid_channel_no((x))	\
					 : external_pid_channel_no((x)))
#define is_pid(x)			(is_internal_pid((x))		\
					 || is_external_pid((x)))
#define is_not_pid(x)			(!is_pid(x))

/*
 * Maximum number of processes. We want the number to fit in a SMALL on
 * 32-bit CPU.
 */

#define ERTS_MAX_PROCESSES		(ERTS_PTAB_MAX_SIZE-1)
#define ERTS_MAX_PID_DATA		((1 << _PID_DATA_SIZE) - 1)
#define ERTS_MAX_PID_NUMBER		((1 << _PID_NUM_SIZE) - 1)
#define ERTS_MAX_PID_SERIAL		((1 << _PID_SER_SIZE) - 1)

#define ERTS_PROC_BITS			(_PID_SER_SIZE + _PID_NUM_SIZE)

#define ERTS_INVALID_PID		ERTS_PTAB_INVALID_ID(_TAG_IMMED1_PID)

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * Ports                                                                   *
\*                                                                         */

extern ErtsPTab erts_port;

#define make_internal_port(D)		erts_ptab_make_id(&erts_port, \
							  (D), \
							  _TAG_IMMED1_PORT)

#define internal_port_index(PRT)	(ASSERT(is_internal_port((PRT))), \
					 erts_ptab_id2pix(&erts_port, (PRT)))

#define internal_port_data(PRT)		(ASSERT(is_internal_port((PRT))), \
					 erts_ptab_id2data(&erts_port, (PRT)))

#define internal_port_number(x) _GET_PORT_NUM(internal_port_data((x)))

#define internal_port_node_name(x)	(internal_port_node((x))->sysname)
#define external_port_node_name(x)	(external_port_node((x))->sysname)
#define internal_port_creation(x)	(internal_port_node((x))->creation)
#define external_port_creation(x)	(external_port_node((x))->creation)
#define internal_port_dist_entry(x)	(internal_port_node((x))->dist_entry)
#define external_port_dist_entry(x)	(external_port_node((x))->dist_entry)

#define internal_port_channel_no(x)	(internal_channel_no((x)))
#define external_port_channel_no(x)	(external_channel_no((x)))

#define port_data_words(x)		(is_internal_port((x))		\
					 ? internal_port_data_words((x))\
					 : external_port_data_words((x)))
#define port_number(x)			(is_internal_port((x))		\
					 ? internal_port_number((x))	\
					 : external_port_number((x)))
#define port_node(x)			(is_internal_port((x))		\
					 ? internal_port_node((x))	\
					 : external_port_node((x)))
#define port_node_name(x)		(is_internal_port((x))		\
					 ? internal_port_node_name((x))	\
					 : external_port_node_name((x)))
#define port_creation(x)		(is_internal_port((x))		\
					 ? internal_port_creation((x))	\
					 : external_port_creation((x)))
#define port_dist_entry(x)		(is_internal_port((x))		\
					 ? internal_port_dist_entry((x))\
					 : external_port_dist_entry((x)))
#define port_channel_no(x)		(is_internal_port((x))		\
					 ? internal_port_channel_no((x))\
					 : external_port_channel_no((x)))

#define is_port(x)			(is_internal_port((x))		\
					 || is_external_port((x)))
#define is_not_port(x)			(!is_port(x))

/* Highest port-ID part in a term of type Port 
   Not necessarily the same as current maximum port table size
   which defines the maximum number of simultaneous Ports
   in the Erlang node. ERTS_MAX_PORTS is a hard upper limit.
*/
#define ERTS_MAX_PORTS			(ERTS_PTAB_MAX_SIZE-1)
#define ERTS_MAX_PORT_DATA		((1 << _PORT_DATA_SIZE) - 1)
#define ERTS_MAX_PORT_NUMBER		((1 << _PORT_NUM_SIZE) - 1)

#define ERTS_PORTS_BITS			(_PORT_NUM_SIZE)

#define ERTS_INVALID_PORT		ERTS_PTAB_INVALID_ID(_TAG_IMMED1_PORT)

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
 * Refs                                                                    *
\*                                                                         */

#if defined(ARCH_64) && !HALFWORD_HEAP

#define internal_ref_no_of_numbers(x)					\
  (internal_ref_data((x))[0])
#define internal_thing_ref_no_of_numbers(thing)			        \
  (internal_thing_ref_data(thing)[0])
#define internal_ref_numbers(x)						\
  (&internal_ref_data((x))[1])
#define internal_thing_ref_numbers(thing)				\
  (&internal_thing_ref_data(thing)[1])
#define external_ref_no_of_numbers(x)					\
  (external_ref_data((x))[0])
#define external_thing_ref_no_of_numbers(thing)			        \
  (external_thing_ref_data(thing)[0])
#define external_ref_numbers(x)						\
  (&external_ref_data((x))[1])
#define external_thing_ref_numbers(thing)				\
  (&external_thing_ref_data(thing)[1])


#else
# 279 "beam/erl_node_container_utils.h"

#define internal_ref_no_of_numbers(x)	(internal_ref_data_words((x)))
#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t))
#define internal_ref_numbers(x)		(internal_ref_data((x)))
#define internal_thing_ref_numbers(t)   (internal_thing_ref_data(t))
#define external_ref_no_of_numbers(x)	(external_ref_data_words((x)))
#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t)))
#define external_ref_numbers(x)		(external_ref_data((x)))
#define external_thing_ref_numbers(t)   (external_thing_ref_data((t)))

#endif
# 290 "beam/erl_node_container_utils.h"

#define internal_ref_node_name(x)	(internal_ref_node((x))->sysname)
#define external_ref_node_name(x)	(external_ref_node((x))->sysname)
#define internal_ref_creation(x)	(internal_ref_node((x))->creation)
#define external_ref_creation(x)	(external_ref_node((x))->creation)
#define internal_ref_dist_entry(x)	(internal_ref_node((x))->dist_entry)
#define external_ref_dist_entry(x)	(external_ref_node((x))->dist_entry)


#define internal_ref_channel_no(x)	(internal_channel_no((x)))
#define external_ref_channel_no(x)	(external_channel_no((x)))

#define ref_data_words(x)		(is_internal_ref((x))		\
					 ? internal_ref_data_words((x))	\
					 : external_ref_data_words((x)))
#define ref_data(x)			(is_internal_ref((x))		\
					 ? internal_ref_data((x))	\
					 : external_ref_data((x)))
#define ref_no_of_numbers(x)		(is_internal_ref((x))		\
					 ? internal_ref_no_of_numbers((x))\
					 : external_ref_no_of_numbers((x)))
#define ref_numbers(x)			(is_internal_ref((x))		\
					 ? internal_ref_numbers((x))	\
					 : external_ref_numbers((x)))
#define ref_node(x)			(is_internal_ref((x))		\
					 ? internal_ref_node(x)		\
					 : external_ref_node((x)))
#define ref_node_name(x)		(is_internal_ref((x))		\
					 ? internal_ref_node_name((x))	\
					 : external_ref_node_name((x)))
#define ref_creation(x)			(is_internal_ref((x))		\
					 ? internal_ref_creation((x))	\
					 : external_ref_creation((x)))
#define ref_dist_entry(x)		(is_internal_ref((x))		\
					 ? internal_ref_dist_entry((x))	\
					 : external_ref_dist_entry((x)))
#define ref_channel_no(x)		(is_internal_ref((x))		\
					 ? internal_ref_channel_no((x))	\
					 : external_ref_channel_no((x)))
#define is_ref(x)			(is_internal_ref((x))		\
					 || is_external_ref((x)))
#define is_ref_rel(x,Base)		(is_internal_ref_rel((x),Base)	\
					 || is_external_ref_rel((x),Base))
#define is_not_ref(x)			(!is_ref(x))

#endif
# 336 "beam/erl_node_container_utils.h"


# 29 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "hash.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/global.h"
# 1 "beam/hash.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 1996-2009. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
** General hash functions
**
*/
#ifndef __HASH_H__
#define __HASH_H__

#ifndef __SYS_H__
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/hash.h"
# 30 "beam/hash.h"
#endif
# 31 "beam/hash.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/hash.h"
# 33 "beam/hash.h"

typedef unsigned long HashValue;

typedef int (*HCMP_FUN)(void*, void*);
typedef HashValue (*H_FUN)(void*);
typedef void* (*HALLOC_FUN)(void*);
typedef void (*HFREE_FUN)(void*);

/*
** This bucket must be placed in top of 
** every object that uses hashing!!!
** (Object*) == (Object*) &bucket
*/
typedef struct hash_bucket
{
    struct hash_bucket* next;	/* Next bucket */
    HashValue hvalue;           /* Store hash value for get, rehash */
} HashBucket;

typedef struct hash_functions
{
    H_FUN hash;
    HCMP_FUN cmp;
    HALLOC_FUN alloc;
    HFREE_FUN free;
} HashFunctions;

typedef struct {
  char *name;
  int   size;
  int   used;
  int   objs;
  int   depth;
} HashInfo;

typedef struct hash
{
    HashFunctions fun;   /* Function block */
    int is_allocated;    /* 0 iff hash structure is on stack or is static */
    ErtsAlcType_t type;
    char* name;          /* Table name (static string, for debugging) */
    int size;		 /* Number of slots */
    int size20percent;   /* 20 percent of number of slots */
    int size80percent;   /* 80 percent of number of slots */
    int ix;              /* Size index in size table */
    int used;		 /* Number of slots used */
    HashBucket** bucket; /* Vector of bucket pointers (objects) */
} Hash;

Hash* hash_new(ErtsAlcType_t, char*, int, HashFunctions);
Hash* hash_init(ErtsAlcType_t, Hash*, char*, int, HashFunctions);

void  hash_delete(Hash*);
void  hash_get_info(HashInfo*, Hash*);
void  hash_info(int, void *, Hash*);
int   hash_table_sz(Hash *);

void* hash_get(Hash*, void*);
void* hash_put(Hash*, void*);
void* hash_erase(Hash*, void*);
void* hash_remove(Hash*, void*);
void  hash_foreach(Hash*, void (*func)(void *, void *), void *);

void erts_hash_merge(Hash* src, Hash* dst);

#endif
# 99 "beam/hash.h"
# 30 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "index.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/global.h"
# 1 "beam/index.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
** General hash and index functions
** The idea behind this file was to capture the
** way Atom,Export and Module table was implemented
*/
#ifndef __INDEX_H__
#define __INDEX_H__

#ifndef __HASH_H__
#if 0 /* expanded by -frewrite-includes */
#include "hash.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/index.h"
# 31 "beam/index.h"
#endif
# 32 "beam/index.h"

typedef struct index_slot 
{
    HashBucket bucket;
    int index;
} IndexSlot;


typedef struct index_table
{
    Hash htable;		/* Mapping obj -> index */
    ErtsAlcType_t type;
    int size;			/* Allocated size */
    int limit;			/* Max size */
    int entries;		/* Number of entries */
    IndexSlot*** seg_table;	/* Mapping index -> obj */
} IndexTable;

#define INDEX_PAGE_SHIFT 10
#define INDEX_PAGE_SIZE (1 << INDEX_PAGE_SHIFT)
#define INDEX_PAGE_MASK ((1 << INDEX_PAGE_SHIFT)-1)

IndexTable *erts_index_init(ErtsAlcType_t,IndexTable*,char*,int,int,HashFunctions);
void index_info(int, void *, IndexTable*);
int index_table_sz(IndexTable *);

int index_get(IndexTable*, void*);

IndexSlot* index_put_entry(IndexTable*, void*);
void erts_index_merge(Hash*, IndexTable*);

/* Erase all entries with index 'ix' and higher
*/
void index_erase_latest_from(IndexTable*, Uint ix);

ERTS_GLB_INLINE int index_put(IndexTable*, void*);
ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int index_put(IndexTable* t, void* tmpl)
{
    return index_put_entry(t, tmpl)->index;
}

ERTS_GLB_INLINE IndexSlot*
erts_index_lookup(IndexTable* t, Uint ix)
{
    return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK];
}
#endif
# 83 "beam/index.h"

#endif
# 85 "beam/index.h"
# 31 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "atom.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/global.h"
# 1 "beam/atom.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ATOM_H__
#define __ATOM_H__

#ifndef __INDEX_H__
#if 0 /* expanded by -frewrite-includes */
#include "index.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/atom.h"
# 26 "beam/atom.h"
#endif
# 27 "beam/atom.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/atom.h"
# 1 "armv6-portbld-freebsd11.0/erl_atom_table.h" 1
/*
 *  Warning: Do not edit this file.  It was automatically
 *  generated by 'make_tables' on Thu May 12 14:27:46 2016.
 */

#ifndef __ERL_ATOM_TABLE_H__
#define __ERL_ATOM_TABLE_H__
extern char* erl_atom_names[];

#define am_false make_atom(0)
#define am_true make_atom(1)
#define am_Underscore make_atom(2)
#define am_Noname make_atom(3)
#define am_EOT make_atom(4)
#define am_Cookie make_atom(5)
#define am_fun make_atom(6)
#define am_infinity make_atom(7)
#define am_timeout make_atom(8)
#define am_normal make_atom(9)
#define am_call make_atom(10)
#define am_return make_atom(11)
#define am_throw make_atom(12)
#define am_error make_atom(13)
#define am_exit make_atom(14)
#define am_undefined make_atom(15)
#define am_nocatch make_atom(16)
#define am_undefined_function make_atom(17)
#define am_undefined_lambda make_atom(18)
#define am_DOWN make_atom(19)
#define am_UP make_atom(20)
#define am_EXIT make_atom(21)
#define am_aborted make_atom(22)
#define am_abs_path make_atom(23)
#define am_absoluteURI make_atom(24)
#define am_ac make_atom(25)
#define am_accessor make_atom(26)
#define am_active make_atom(27)
#define am_active_tasks make_atom(28)
#define am_all make_atom(29)
#define am_all_but_first make_atom(30)
#define am_all_names make_atom(31)
#define am_alloc_info make_atom(32)
#define am_alloc_sizes make_atom(33)
#define am_allocated make_atom(34)
#define am_allocated_areas make_atom(35)
#define am_allocator make_atom(36)
#define am_allocator_sizes make_atom(37)
#define am_alloc_util_allocators make_atom(38)
#define am_allow_gc make_atom(39)
#define am_allow_passive_connect make_atom(40)
#define am_already_loaded make_atom(41)
#define am_amd64 make_atom(42)
#define am_anchored make_atom(43)
#define am_and make_atom(44)
#define am_andalso make_atom(45)
#define am_andthen make_atom(46)
#define am_any make_atom(47)
#define am_anycrlf make_atom(48)
#define am_apply make_atom(49)
#define am_args make_atom(50)
#define am_arg0 make_atom(51)
#define am_arity make_atom(52)
#define am_asn1 make_atom(53)
#define am_async make_atom(54)
#define am_asynchronous make_atom(55)
#define am_atom make_atom(56)
#define am_atom_used make_atom(57)
#define am_attributes make_atom(58)
#define am_await_port_send_result make_atom(59)
#define am_await_proc_exit make_atom(60)
#define am_await_result make_atom(61)
#define am_await_sched_wall_time_modifications make_atom(62)
#define am_awaiting_load make_atom(63)
#define am_awaiting_unload make_atom(64)
#define am_backtrace make_atom(65)
#define am_backtrace_depth make_atom(66)
#define am_badarg make_atom(67)
#define am_badarith make_atom(68)
#define am_badarity make_atom(69)
#define am_badfile make_atom(70)
#define am_badfun make_atom(71)
#define am_badkey make_atom(72)
#define am_badmap make_atom(73)
#define am_badmatch make_atom(74)
#define am_badsig make_atom(75)
#define am_bag make_atom(76)
#define am_band make_atom(77)
#define am_big make_atom(78)
#define am_bif_return_trap make_atom(79)
#define am_bif_timer_server make_atom(80)
#define am_binary make_atom(81)
#define am_binary_bin_to_list_trap make_atom(82)
#define am_binary_copy_trap make_atom(83)
#define am_binary_longest_prefix_trap make_atom(84)
#define am_binary_longest_suffix_trap make_atom(85)
#define am_binary_match_trap make_atom(86)
#define am_binary_matches_trap make_atom(87)
#define am_binary_to_list_continue make_atom(88)
#define am_binary_to_term_trap make_atom(89)
#define am_block make_atom(90)
#define am_blocked make_atom(91)
#define am_bm make_atom(92)
#define am_bnot make_atom(93)
#define am_bor make_atom(94)
#define am_bxor make_atom(95)
#define am_break_ignored make_atom(96)
#define am_breakpoint make_atom(97)
#define am_bsl make_atom(98)
#define am_bsr make_atom(99)
#define am_bsr_anycrlf make_atom(100)
#define am_bsr_unicode make_atom(101)
#define am_build_type make_atom(102)
#define am_busy_dist_port make_atom(103)
#define am_busy_port make_atom(104)
#define am_call_count make_atom(105)
#define am_call_time make_atom(106)
#define am_caller make_atom(107)
#define am_capture make_atom(108)
#define am_case_clause make_atom(109)
#define am_caseless make_atom(110)
#define am_catchlevel make_atom(111)
#define am_cd make_atom(112)
#define am_cdr make_atom(113)
#define am_cflags make_atom(114)
#define am_CHANGE make_atom(115)
#define am_characters_to_binary_int make_atom(116)
#define am_characters_to_list_int make_atom(117)
#define am_clear make_atom(118)
#define am_clock_service make_atom(119)
#define am_close make_atom(120)
#define am_closed make_atom(121)
#define am_code make_atom(122)
#define am_command make_atom(123)
#define am_compact make_atom(124)
#define am_compat_rel make_atom(125)
#define am_compile make_atom(126)
#define am_compressed make_atom(127)
#define am_config_h make_atom(128)
#define am_convert_time_unit make_atom(129)
#define am_connect make_atom(130)
#define am_connected make_atom(131)
#define am_connection_closed make_atom(132)
#define am_cons make_atom(133)
#define am_const make_atom(134)
#define am_context_switches make_atom(135)
#define am_control make_atom(136)
#define am_copy make_atom(137)
#define am_cpu make_atom(138)
#define am_cpu_timestamp make_atom(139)
#define am_cr make_atom(140)
#define am_crlf make_atom(141)
#define am_creation make_atom(142)
#define am_current_function make_atom(143)
#define am_current_location make_atom(144)
#define am_current_stacktrace make_atom(145)
#define am_data make_atom(146)
#define am_debug_flags make_atom(147)
#define am_decimals make_atom(148)
#define am_delay_trap make_atom(149)
#define am_dexit make_atom(150)
#define am_depth make_atom(151)
#define am_dgroup_leader make_atom(152)
#define am_dictionary make_atom(153)
#define am_dirty_cpu_schedulers_online make_atom(154)
#define am_disable_trace make_atom(155)
#define am_disabled make_atom(156)
#define am_display_items make_atom(157)
#define am_dist make_atom(158)
#define am_dist_cmd make_atom(159)
#define am_Div make_atom(160)
#define am_div make_atom(161)
#define am_dlink make_atom(162)
#define am_dmonitor_node make_atom(163)
#define am_dmonitor_p make_atom(164)
#define am_DollarDollar make_atom(165)
#define am_DollarUnderscore make_atom(166)
#define am_dollar_endonly make_atom(167)
#define am_dotall make_atom(168)
#define am_driver make_atom(169)
#define am_driver_options make_atom(170)
#define am_dsend make_atom(171)
#define am_dsend_continue_trap make_atom(172)
#define am_dunlink make_atom(173)
#define am_duplicate_bag make_atom(174)
#define am_dupnames make_atom(175)
#define am_elib_malloc make_atom(176)
#define am_emulator make_atom(177)
#define am_enable_trace make_atom(178)
#define am_enabled make_atom(179)
#define am_endian make_atom(180)
#define am_env make_atom(181)
#define am_eof make_atom(182)
#define am_eol make_atom(183)
#define am_exception_from make_atom(184)
#define am_exception_trace make_atom(185)
#define am_extended make_atom(186)
#define am_Eq make_atom(187)
#define am_Eqeq make_atom(188)
#define am_erlang make_atom(189)
#define am_ERROR make_atom(190)
#define am_error_handler make_atom(191)
#define am_error_logger make_atom(192)
#define am_erts_internal make_atom(193)
#define am_ets make_atom(194)
#define am_ETS_TRANSFER make_atom(195)
#define am_event make_atom(196)
#define am_exact_reductions make_atom(197)
#define am_exclusive make_atom(198)
#define am_exit_status make_atom(199)
#define am_existing make_atom(200)
#define am_exiting make_atom(201)
#define am_exports make_atom(202)
#define am_external make_atom(203)
#define am_fcgi make_atom(204)
#define am_fd make_atom(205)
#define am_first make_atom(206)
#define am_firstline make_atom(207)
#define am_flags make_atom(208)
#define am_flush make_atom(209)
#define am_flush_monitor_messages make_atom(210)
#define am_force make_atom(211)
#define am_format_cpu_topology make_atom(212)
#define am_free make_atom(213)
#define am_fullsweep_after make_atom(214)
#define am_fullsweep_if_old_binaries make_atom(215)
#define am_function make_atom(216)
#define am_functions make_atom(217)
#define am_function_clause make_atom(218)
#define am_garbage_collecting make_atom(219)
#define am_garbage_collection make_atom(220)
#define am_gc_end make_atom(221)
#define am_gc_start make_atom(222)
#define am_Ge make_atom(223)
#define am_generational make_atom(224)
#define am_get_data make_atom(225)
#define am_get_seq_token make_atom(226)
#define am_get_tcw make_atom(227)
#define am_getenv make_atom(228)
#define am_gather_gc_info_result make_atom(229)
#define am_gather_io_bytes make_atom(230)
#define am_gather_sched_wall_time_result make_atom(231)
#define am_gather_system_check_result make_atom(232)
#define am_getting_linked make_atom(233)
#define am_getting_unlinked make_atom(234)
#define am_global make_atom(235)
#define am_Gt make_atom(236)
#define am_grun make_atom(237)
#define am_group_leader make_atom(238)
#define am_have_dt_utag make_atom(239)
#define am_heap_block_size make_atom(240)
#define am_heap_size make_atom(241)
#define am_heap_sizes make_atom(242)
#define am_heap_type make_atom(243)
#define am_heart_port make_atom(244)
#define am_heir make_atom(245)
#define am_hidden make_atom(246)
#define am_hide make_atom(247)
#define am_high make_atom(248)
#define am_hipe_architecture make_atom(249)
#define am_http make_atom(250)
#define am_httph make_atom(251)
#define am_https make_atom(252)
#define am_http_response make_atom(253)
#define am_http_request make_atom(254)
#define am_http_header make_atom(255)
#define am_http_eoh make_atom(256)
#define am_http_error make_atom(257)
#define am_http_bin make_atom(258)
#define am_httph_bin make_atom(259)
#define am_id make_atom(260)
#define am_if_clause make_atom(261)
#define am_ignore make_atom(262)
#define am_in make_atom(263)
#define am_in_exiting make_atom(264)
#define am_inactive make_atom(265)
#define am_incomplete make_atom(266)
#define am_inconsistent make_atom(267)
#define am_index make_atom(268)
#define am_info make_atom(269)
#define am_info_msg make_atom(270)
#define am_initial_call make_atom(271)
#define am_input make_atom(272)
#define am_internal make_atom(273)
#define am_internal_error make_atom(274)
#define am_internal_status make_atom(275)
#define am_instruction_counts make_atom(276)
#define am_invalid make_atom(277)
#define am_is_constant make_atom(278)
#define am_is_seq_trace make_atom(279)
#define am_io make_atom(280)
#define am_keypos make_atom(281)
#define am_kill make_atom(282)
#define am_killed make_atom(283)
#define am_kill_ports make_atom(284)
#define am_known make_atom(285)
#define am_label make_atom(286)
#define am_large_heap make_atom(287)
#define am_last_calls make_atom(288)
#define am_latin1 make_atom(289)
#define am_ldflags make_atom(290)
#define am_Le make_atom(291)
#define am_lf make_atom(292)
#define am_line make_atom(293)
#define am_line_delimiter make_atom(294)
#define am_line_length make_atom(295)
#define am_linked_in_driver make_atom(296)
#define am_links make_atom(297)
#define am_list make_atom(298)
#define am_list_to_binary_continue make_atom(299)
#define am_little make_atom(300)
#define am_loaded make_atom(301)
#define am_load_cancelled make_atom(302)
#define am_load_failure make_atom(303)
#define am_local make_atom(304)
#define am_long_gc make_atom(305)
#define am_long_schedule make_atom(306)
#define am_low make_atom(307)
#define am_Lt make_atom(308)
#define am_machine make_atom(309)
#define am_match make_atom(310)
#define am_match_limit make_atom(311)
#define am_match_limit_recursion make_atom(312)
#define am_match_spec make_atom(313)
#define am_max make_atom(314)
#define am_maximum make_atom(315)
#define am_max_tables make_atom(316)
#define am_max_processes make_atom(317)
#define am_mbuf_size make_atom(318)
#define am_md5 make_atom(319)
#define am_memory make_atom(320)
#define am_memory_internal make_atom(321)
#define am_memory_types make_atom(322)
#define am_message make_atom(323)
#define am_message_binary make_atom(324)
#define am_message_queue_len make_atom(325)
#define am_messages make_atom(326)
#define am_merge_trap make_atom(327)
#define am_meta make_atom(328)
#define am_meta_match_spec make_atom(329)
#define am_micro_seconds make_atom(330)
#define am_milli_seconds make_atom(331)
#define am_min_heap_size make_atom(332)
#define am_min_bin_vheap_size make_atom(333)
#define am_minor_version make_atom(334)
#define am_Minus make_atom(335)
#define am_module make_atom(336)
#define am_module_info make_atom(337)
#define am_monitored_by make_atom(338)
#define am_monitor make_atom(339)
#define am_monitor_nodes make_atom(340)
#define am_monitors make_atom(341)
#define am_monotonic make_atom(342)
#define am_monotonic_timestamp make_atom(343)
#define am_more make_atom(344)
#define am_multi_scheduling make_atom(345)
#define am_multiline make_atom(346)
#define am_nano_seconds make_atom(347)
#define am_name make_atom(348)
#define am_named_table make_atom(349)
#define am_namelist make_atom(350)
#define am_native make_atom(351)
#define am_native_addresses make_atom(352)
#define am_Neq make_atom(353)
#define am_Neqeq make_atom(354)
#define am_net_kernel make_atom(355)
#define am_net_kernel_terminated make_atom(356)
#define am_never_utf make_atom(357)
#define am_new make_atom(358)
#define am_new_index make_atom(359)
#define am_new_uniq make_atom(360)
#define am_newline make_atom(361)
#define am_next make_atom(362)
#define am_no make_atom(363)
#define am_nomatch make_atom(364)
#define am_none make_atom(365)
#define am_no_auto_capture make_atom(366)
#define am_noconnect make_atom(367)
#define am_noconnection make_atom(368)
#define am_nocookie make_atom(369)
#define am_node make_atom(370)
#define am_node_type make_atom(371)
#define am_nodedown make_atom(372)
#define am_nodedown_reason make_atom(373)
#define am_nodeup make_atom(374)
#define am_noeol make_atom(375)
#define am_nofile make_atom(376)
#define am_noproc make_atom(377)
#define am_nosuspend make_atom(378)
#define am_no_float make_atom(379)
#define am_no_integer make_atom(380)
#define am_no_network make_atom(381)
#define am_no_start_optimize make_atom(382)
#define am_not make_atom(383)
#define am_not_a_list make_atom(384)
#define am_not_loaded make_atom(385)
#define am_not_loaded_by_this_process make_atom(386)
#define am_not_pending make_atom(387)
#define am_not_purged make_atom(388)
#define am_notalive make_atom(389)
#define am_notbol make_atom(390)
#define am_noteol make_atom(391)
#define am_notempty make_atom(392)
#define am_notempty_atstart make_atom(393)
#define am_notify make_atom(394)
#define am_notsup make_atom(395)
#define am_nouse_stdio make_atom(396)
#define am_objects make_atom(397)
#define am_offset make_atom(398)
#define am_ok make_atom(399)
#define am_old_heap_block_size make_atom(400)
#define am_old_heap_size make_atom(401)
#define am_on_load make_atom(402)
#define am_open make_atom(403)
#define am_open_error make_atom(404)
#define am_opt make_atom(405)
#define am_or make_atom(406)
#define am_ordered_set make_atom(407)
#define am_orelse make_atom(408)
#define am_os_pid make_atom(409)
#define am_os_type make_atom(410)
#define am_os_version make_atom(411)
#define am_ose_bg_proc make_atom(412)
#define am_ose_int_proc make_atom(413)
#define am_ose_phantom make_atom(414)
#define am_ose_pri_proc make_atom(415)
#define am_ose_process_prio make_atom(416)
#define am_ose_process_type make_atom(417)
#define am_ose_ti_proc make_atom(418)
#define am_out make_atom(419)
#define am_out_exited make_atom(420)
#define am_out_exiting make_atom(421)
#define am_output make_atom(422)
#define am_overlapped_io make_atom(423)
#define am_owner make_atom(424)
#define am_packet make_atom(425)
#define am_packet_size make_atom(426)
#define am_parallelism make_atom(427)
#define am_Plus make_atom(428)
#define am_pause make_atom(429)
#define am_pending make_atom(430)
#define am_pending_driver make_atom(431)
#define am_pending_process make_atom(432)
#define am_pending_reload make_atom(433)
#define am_permanent make_atom(434)
#define am_pid make_atom(435)
#define am_port make_atom(436)
#define am_ports make_atom(437)
#define am_port_count make_atom(438)
#define am_port_limit make_atom(439)
#define am_port_op make_atom(440)
#define am_positive make_atom(441)
#define am_print make_atom(442)
#define am_priority make_atom(443)
#define am_private make_atom(444)
#define am_process make_atom(445)
#define am_processes make_atom(446)
#define am_processes_used make_atom(447)
#define am_process_count make_atom(448)
#define am_process_display make_atom(449)
#define am_process_limit make_atom(450)
#define am_process_dump make_atom(451)
#define am_procs make_atom(452)
#define am_proc_sig make_atom(453)
#define am_profile make_atom(454)
#define am_protected make_atom(455)
#define am_protection make_atom(456)
#define am_ptab_list_continue make_atom(457)
#define am_public make_atom(458)
#define am_purify make_atom(459)
#define am_quantify make_atom(460)
#define am_queue_size make_atom(461)
#define am_raw make_atom(462)
#define am_re make_atom(463)
#define am_re_pattern make_atom(464)
#define am_re_run_trap make_atom(465)
#define am_read_concurrency make_atom(466)
#define am_ready_input make_atom(467)
#define am_ready_output make_atom(468)
#define am_ready_async make_atom(469)
#define am_reason make_atom(470)
#define am_receive make_atom(471)
#define am_recent_size make_atom(472)
#define am_reductions make_atom(473)
#define am_refc make_atom(474)
#define am_register make_atom(475)
#define am_registered_name make_atom(476)
#define am_reload make_atom(477)
#define am_rem make_atom(478)
#define am_report_errors make_atom(479)
#define am_reset make_atom(480)
#define am_restart make_atom(481)
#define am_return_from make_atom(482)
#define am_return_to make_atom(483)
#define am_return_trace make_atom(484)
#define am_run_queue make_atom(485)
#define am_run_queue_lengths make_atom(486)
#define am_runnable make_atom(487)
#define am_runnable_ports make_atom(488)
#define am_runnable_procs make_atom(489)
#define am_running make_atom(490)
#define am_running_ports make_atom(491)
#define am_running_procs make_atom(492)
#define am_runtime make_atom(493)
#define am_safe make_atom(494)
#define am_save_calls make_atom(495)
#define am_scheduler make_atom(496)
#define am_scheduler_id make_atom(497)
#define am_schedulers_online make_atom(498)
#define am_scheme make_atom(499)
#define am_scientific make_atom(500)
#define am_scope make_atom(501)
#define am_seconds make_atom(502)
#define am_sensitive make_atom(503)
#define am_sequential_tracer make_atom(504)
#define am_sequential_trace_token make_atom(505)
#define am_serial make_atom(506)
#define am_set make_atom(507)
#define am_set_cpu_topology make_atom(508)
#define am_set_data make_atom(509)
#define am_set_on_first_link make_atom(510)
#define am_set_on_first_spawn make_atom(511)
#define am_set_on_link make_atom(512)
#define am_set_on_spawn make_atom(513)
#define am_set_seq_token make_atom(514)
#define am_set_tcw make_atom(515)
#define am_set_tcw_fake make_atom(516)
#define am_separate make_atom(517)
#define am_shared make_atom(518)
#define am_silent make_atom(519)
#define am_size make_atom(520)
#define am_sl_alloc make_atom(521)
#define am_spawn_executable make_atom(522)
#define am_spawn_driver make_atom(523)
#define am_ssl_tls make_atom(524)
#define am_stack_size make_atom(525)
#define am_start make_atom(526)
#define am_status make_atom(527)
#define am_static make_atom(528)
#define am_stderr_to_stdout make_atom(529)
#define am_stop make_atom(530)
#define am_stream make_atom(531)
#define am_strict_monotonic_timestamp make_atom(532)
#define am_sunrm make_atom(533)
#define am_suspend make_atom(534)
#define am_suspended make_atom(535)
#define am_suspending make_atom(536)
#define am_sys_misc make_atom(537)
#define am_system make_atom(538)
#define am_system_error make_atom(539)
#define am_system_limit make_atom(540)
#define am_system_version make_atom(541)
#define am_system_architecture make_atom(542)
#define am_SYSTEM make_atom(543)
#define am_table make_atom(544)
#define am_term_to_binary_trap make_atom(545)
#define am_this make_atom(546)
#define am_thread_pool_size make_atom(547)
#define am_threads make_atom(548)
#define am_time_offset make_atom(549)
#define am_timeout_value make_atom(550)
#define am_Times make_atom(551)
#define am_timestamp make_atom(552)
#define am_total make_atom(553)
#define am_total_active_tasks make_atom(554)
#define am_total_heap_size make_atom(555)
#define am_total_run_queue_lengths make_atom(556)
#define am_tpkt make_atom(557)
#define am_trace make_atom(558)
#define am_trace_ts make_atom(559)
#define am_traced make_atom(560)
#define am_trace_control_word make_atom(561)
#define am_tracer make_atom(562)
#define am_trap_exit make_atom(563)
#define am_try_clause make_atom(564)
#define am_tuple make_atom(565)
#define am_type make_atom(566)
#define am_ucompile make_atom(567)
#define am_ucp make_atom(568)
#define am_undef make_atom(569)
#define am_ungreedy make_atom(570)
#define am_unicode make_atom(571)
#define am_unregister make_atom(572)
#define am_urun make_atom(573)
#define am_use_stdio make_atom(574)
#define am_used make_atom(575)
#define am_utf8 make_atom(576)
#define am_unblock make_atom(577)
#define am_uniq make_atom(578)
#define am_unless_suspending make_atom(579)
#define am_unloaded make_atom(580)
#define am_unloading make_atom(581)
#define am_unloaded_only make_atom(582)
#define am_unload_cancelled make_atom(583)
#define am_value make_atom(584)
#define am_values make_atom(585)
#define am_version make_atom(586)
#define am_visible make_atom(587)
#define am_waiting make_atom(588)
#define am_wall_clock make_atom(589)
#define am_warning make_atom(590)
#define am_warning_msg make_atom(591)
#define am_scheduler_wall_time make_atom(592)
#define am_wordsize make_atom(593)
#define am_write_concurrency make_atom(594)
#define am_xor make_atom(595)
#define am_x86 make_atom(596)
#define am_yes make_atom(597)
#define am_yield make_atom(598)
#define am_abs make_atom(599)
#define am_adler32 make_atom(600)
#define am_adler32_combine make_atom(601)
#define am_atom_to_list make_atom(602)
#define am_binary_to_list make_atom(603)
#define am_binary_to_term make_atom(604)
#define am_crc32 make_atom(605)
#define am_crc32_combine make_atom(606)
#define am_date make_atom(607)
#define am_delete_module make_atom(608)
#define am_display make_atom(609)
#define am_display_string make_atom(610)
#define am_display_nl make_atom(611)
#define am_element make_atom(612)
#define am_erase make_atom(613)
#define am_external_size make_atom(614)
#define am_float make_atom(615)
#define am_float_to_list make_atom(616)
#define am_fun_info make_atom(617)
#define am_garbage_collect make_atom(618)
#define am_get make_atom(619)
#define am_get_keys make_atom(620)
#define am_halt make_atom(621)
#define am_phash make_atom(622)
#define am_phash2 make_atom(623)
#define am_hd make_atom(624)
#define am_integer_to_list make_atom(625)
#define am_is_alive make_atom(626)
#define am_length make_atom(627)
#define am_link make_atom(628)
#define am_list_to_atom make_atom(629)
#define am_list_to_binary make_atom(630)
#define am_list_to_float make_atom(631)
#define am_list_to_integer make_atom(632)
#define am_list_to_pid make_atom(633)
#define am_list_to_tuple make_atom(634)
#define am_localtime make_atom(635)
#define am_localtime_to_universaltime make_atom(636)
#define am_make_ref make_atom(637)
#define am_unique_integer make_atom(638)
#define am_md5_init make_atom(639)
#define am_md5_update make_atom(640)
#define am_md5_final make_atom(641)
#define am_module_loaded make_atom(642)
#define am_function_exported make_atom(643)
#define am_monitor_node make_atom(644)
#define am_nodes make_atom(645)
#define am_now make_atom(646)
#define am_monotonic_time make_atom(647)
#define am_system_time make_atom(648)
#define am_open_port make_atom(649)
#define am_pid_to_list make_atom(650)
#define am_pre_loaded make_atom(651)
#define am_process_flag make_atom(652)
#define am_process_info make_atom(653)
#define am_purge_module make_atom(654)
#define am_put make_atom(655)
#define am_registered make_atom(656)
#define am_round make_atom(657)
#define am_self make_atom(658)
#define am_setelement make_atom(659)
#define am_spawn make_atom(660)
#define am_spawn_link make_atom(661)
#define am_split_binary make_atom(662)
#define am_statistics make_atom(663)
#define am_term_to_binary make_atom(664)
#define am_time make_atom(665)
#define am_tl make_atom(666)
#define am_trunc make_atom(667)
#define am_tuple_to_list make_atom(668)
#define am_universaltime make_atom(669)
#define am_universaltime_to_localtime make_atom(670)
#define am_unlink make_atom(671)
#define am_whereis make_atom(672)
#define am_spawn_opt make_atom(673)
#define am_setnode make_atom(674)
#define am_dist_exit make_atom(675)
#define am_port_info make_atom(676)
#define am_port_call make_atom(677)
#define am_port_command make_atom(678)
#define am_port_control make_atom(679)
#define am_port_close make_atom(680)
#define am_port_connect make_atom(681)
#define am_request_system_task make_atom(682)
#define am_check_process_code make_atom(683)
#define am_map_to_tuple_keys make_atom(684)
#define am_map_type make_atom(685)
#define am_map_hashmap_children make_atom(686)
#define am_time_unit make_atom(687)
#define am_is_system_process make_atom(688)
#define am_system_check make_atom(689)
#define am_port_set_data make_atom(690)
#define am_port_get_data make_atom(691)
#define am_trace_pattern make_atom(692)
#define am_trace_info make_atom(693)
#define am_trace_delivered make_atom(694)
#define am_seq_trace make_atom(695)
#define am_seq_trace_info make_atom(696)
#define am_seq_trace_print make_atom(697)
#define am_suspend_process make_atom(698)
#define am_resume_process make_atom(699)
#define am_bump_reductions make_atom(700)
#define am_math make_atom(701)
#define am_cos make_atom(702)
#define am_cosh make_atom(703)
#define am_sin make_atom(704)
#define am_sinh make_atom(705)
#define am_tan make_atom(706)
#define am_tanh make_atom(707)
#define am_acos make_atom(708)
#define am_acosh make_atom(709)
#define am_asin make_atom(710)
#define am_asinh make_atom(711)
#define am_atan make_atom(712)
#define am_atanh make_atom(713)
#define am_erf make_atom(714)
#define am_erfc make_atom(715)
#define am_exp make_atom(716)
#define am_log make_atom(717)
#define am_log2 make_atom(718)
#define am_log10 make_atom(719)
#define am_sqrt make_atom(720)
#define am_atan2 make_atom(721)
#define am_pow make_atom(722)
#define am_start_timer make_atom(723)
#define am_send_after make_atom(724)
#define am_cancel_timer make_atom(725)
#define am_read_timer make_atom(726)
#define am_make_tuple make_atom(727)
#define am_append_element make_atom(728)
#define am_system_flag make_atom(729)
#define am_system_info make_atom(730)
#define am_system_monitor make_atom(731)
#define am_system_profile make_atom(732)
#define am_ref_to_list make_atom(733)
#define am_port_to_list make_atom(734)
#define am_fun_to_list make_atom(735)
#define am_demonitor make_atom(736)
#define am_is_process_alive make_atom(737)
#define am_raise make_atom(738)
#define am_get_stacktrace make_atom(739)
#define am_is_builtin make_atom(740)
#define am__AtomAlias26 make_atom(741)
#define am_send make_atom(742)
#define am__AtomAlias27 make_atom(743)
#define am_append make_atom(744)
#define am__AtomAlias28 make_atom(745)
#define am_subtract make_atom(746)
#define am_is_atom make_atom(747)
#define am_is_list make_atom(748)
#define am_is_tuple make_atom(749)
#define am_is_float make_atom(750)
#define am_is_integer make_atom(751)
#define am_is_number make_atom(752)
#define am_is_pid make_atom(753)
#define am_is_port make_atom(754)
#define am_is_reference make_atom(755)
#define am_is_binary make_atom(756)
#define am_is_function make_atom(757)
#define am_is_record make_atom(758)
#define am_match_spec_test make_atom(759)
#define am_delete make_atom(760)
#define am_delete_all_objects make_atom(761)
#define am_delete_object make_atom(762)
#define am_is_compiled_ms make_atom(763)
#define am_lookup make_atom(764)
#define am_lookup_element make_atom(765)
#define am_last make_atom(766)
#define am_match_object make_atom(767)
#define am_member make_atom(768)
#define am_prev make_atom(769)
#define am_insert make_atom(770)
#define am_insert_new make_atom(771)
#define am_rename make_atom(772)
#define am_safe_fixtable make_atom(773)
#define am_slot make_atom(774)
#define am_update_counter make_atom(775)
#define am_select make_atom(776)
#define am_select_count make_atom(777)
#define am_select_reverse make_atom(778)
#define am_select_delete make_atom(779)
#define am_match_spec_compile make_atom(780)
#define am_match_spec_run_r make_atom(781)
#define am_os make_atom(782)
#define am_putenv make_atom(783)
#define am_getpid make_atom(784)
#define am_erl_ddll make_atom(785)
#define am_try_load make_atom(786)
#define am_try_unload make_atom(787)
#define am_loaded_drivers make_atom(788)
#define am_format_error_int make_atom(789)
#define am_run make_atom(790)
#define am_lists make_atom(791)
#define am_reverse make_atom(792)
#define am_keymember make_atom(793)
#define am_keysearch make_atom(794)
#define am_keyfind make_atom(795)
#define am_erts_debug make_atom(796)
#define am_disassemble make_atom(797)
#define am_same make_atom(798)
#define am_flat_size make_atom(799)
#define am_get_internal_state make_atom(800)
#define am_set_internal_state make_atom(801)
#define am_dist_ext_to_term make_atom(802)
#define am_instructions make_atom(803)
#define am_dump_monitors make_atom(804)
#define am_dump_links make_atom(805)
#define am_lock_counters make_atom(806)
#define am_get_chunk make_atom(807)
#define am_module_md5 make_atom(808)
#define am_make_stub_module make_atom(809)
#define am_is_module_native make_atom(810)
#define am_hibernate make_atom(811)
#define am_warning_map make_atom(812)
#define am_get_module_info make_atom(813)
#define am_is_boolean make_atom(814)
#define am_string make_atom(815)
#define am_to_integer make_atom(816)
#define am_to_float make_atom(817)
#define am_make_fun make_atom(818)
#define am_iolist_size make_atom(819)
#define am_iolist_to_binary make_atom(820)
#define am_list_to_existing_atom make_atom(821)
#define am_is_bitstring make_atom(822)
#define am_tuple_size make_atom(823)
#define am_byte_size make_atom(824)
#define am_bit_size make_atom(825)
#define am_list_to_bitstring make_atom(826)
#define am_bitstring_to_list make_atom(827)
#define am_update_element make_atom(828)
#define am_decode_packet make_atom(829)
#define am_characters_to_binary make_atom(830)
#define am_characters_to_list make_atom(831)
#define am_bin_is_7bit make_atom(832)
#define am_atom_to_binary make_atom(833)
#define am_binary_to_atom make_atom(834)
#define am_binary_to_existing_atom make_atom(835)
#define am_dflag_unicode_io make_atom(836)
#define am_give_away make_atom(837)
#define am_setopts make_atom(838)
#define am_load_nif make_atom(839)
#define am_call_on_load_function make_atom(840)
#define am_finish_after_on_load make_atom(841)
#define am_binary_part make_atom(842)
#define am_compile_pattern make_atom(843)
#define am_matches make_atom(844)
#define am_longest_common_prefix make_atom(845)
#define am_longest_common_suffix make_atom(846)
#define am_at make_atom(847)
#define am_part make_atom(848)
#define am_bin_to_list make_atom(849)
#define am_list_to_bin make_atom(850)
#define am_referenced_byte_size make_atom(851)
#define am_encode_unsigned make_atom(852)
#define am_decode_unsigned make_atom(853)
#define am_nif_error make_atom(854)
#define am_prim_file make_atom(855)
#define am_internal_name2native make_atom(856)
#define am_internal_native2name make_atom(857)
#define am_internal_normalize_utf8 make_atom(858)
#define am_is_translatable make_atom(859)
#define am_file make_atom(860)
#define am_native_name_encoding make_atom(861)
#define am_check_old_code make_atom(862)
#define am_universaltime_to_posixtime make_atom(863)
#define am_posixtime_to_universaltime make_atom(864)
#define am_dt_put_tag make_atom(865)
#define am_dt_get_tag make_atom(866)
#define am_dt_get_tag_data make_atom(867)
#define am_dt_spread_tag make_atom(868)
#define am_dt_restore_tag make_atom(869)
#define am_dt_prepend_vm_tag_data make_atom(870)
#define am_dt_append_vm_tag_data make_atom(871)
#define am_prepare_loading make_atom(872)
#define am_finish_loading make_atom(873)
#define am_insert_element make_atom(874)
#define am_delete_element make_atom(875)
#define am_binary_to_integer make_atom(876)
#define am_integer_to_binary make_atom(877)
#define am_float_to_binary make_atom(878)
#define am_binary_to_float make_atom(879)
#define am_printable_range make_atom(880)
#define am_unsetenv make_atom(881)
#define am_inspect make_atom(882)
#define am_is_map make_atom(883)
#define am_map_size make_atom(884)
#define am_maps make_atom(885)
#define am_to_list make_atom(886)
#define am_find make_atom(887)
#define am_from_list make_atom(888)
#define am_is_key make_atom(889)
#define am_keys make_atom(890)
#define am_merge make_atom(891)
#define am_remove make_atom(892)
#define am_update make_atom(893)
#define am_cmp_term make_atom(894)
#define am_take make_atom(895)
#define am_fun_info_mfa make_atom(896)
#define am_map_info make_atom(897)
#define am_hash make_atom(898)
#define am_hipe_bifs make_atom(899)
#define am_write_u8 make_atom(900)
#define am_write_u32 make_atom(901)
#define am_bytearray make_atom(902)
#define am_bytearray_sub make_atom(903)
#define am_bytearray_update make_atom(904)
#define am_bitarray make_atom(905)
#define am_bitarray_sub make_atom(906)
#define am_bitarray_update make_atom(907)
#define am_array make_atom(908)
#define am_array_length make_atom(909)
#define am_array_sub make_atom(910)
#define am_array_update make_atom(911)
#define am_ref make_atom(912)
#define am_ref_get make_atom(913)
#define am_ref_set make_atom(914)
#define am_enter_code make_atom(915)
#define am_alloc_data make_atom(916)
#define am_constants_size make_atom(917)
#define am_merge_term make_atom(918)
#define am_fun_to_address make_atom(919)
#define am_set_native_address make_atom(920)
#define am_set_funinfo_native_address make_atom(921)
#define am_invalidate_funinfo_native_addresses make_atom(922)
#define am_update_code_size make_atom(923)
#define am_code_size make_atom(924)
#define am_enter_sdesc make_atom(925)
#define am_bif_address make_atom(926)
#define am_primop_address make_atom(927)
#define am_atom_to_word make_atom(928)
#define am_term_to_word make_atom(929)
#define am_get_fe make_atom(930)
#define am_set_native_address_in_fe make_atom(931)
#define am_find_na_or_make_stub make_atom(932)
#define am_check_crc make_atom(933)
#define am_system_crc make_atom(934)
#define am_get_rts_param make_atom(935)
#define am_patch_insn make_atom(936)
#define am_patch_call make_atom(937)
#define am_add_ref make_atom(938)
#define am_mark_referred_from make_atom(939)
#define am_remove_refs_from make_atom(940)
#define am_redirect_referred_from make_atom(941)
#define am_load_mfa make_atom(942)
#define am_remote make_atom(943)
#define am_c_const make_atom(944)
#define am_closure make_atom(945)
#define am_constant make_atom(946)
#define am_x86_abs_pcrel make_atom(947)
#define am_load_fe make_atom(948)
#define am_suspend_msg make_atom(949)
#define am_suspend_msg_timeout make_atom(950)
#define am_suspend_0 make_atom(951)
#define am_gc_1 make_atom(952)
#define am_hipe_apply make_atom(953)
#define am_rethrow make_atom(954)
#define am_nonclosure_address make_atom(955)
#define am_atomic_inc make_atom(956)
#define am_clear_timeout make_atom(957)
#define am_check_get_msg make_atom(958)
#define am_select_msg make_atom(959)
#define am_set_timeout make_atom(960)
#define am_cmp_2 make_atom(961)
#define am_op_exact_eqeq_2 make_atom(962)
#define am_conv_big_to_float make_atom(963)
#define am_fclearerror_error make_atom(964)
#define am_bs_put_big_integer make_atom(965)
#define am_bs_put_small_float make_atom(966)
#define am_bs_put_bits make_atom(967)
#define am_bs_allocate make_atom(968)
#define am_bs_get_integer_2 make_atom(969)
#define am_bs_get_float_2 make_atom(970)
#define am_bs_get_binary_2 make_atom(971)
#define am_bs_reallocate make_atom(972)
#define am_bs_utf8_size make_atom(973)
#define am_bs_put_utf8 make_atom(974)
#define am_bs_get_utf8 make_atom(975)
#define am_bs_utf16_size make_atom(976)
#define am_bs_put_utf16be make_atom(977)
#define am_bs_put_utf16le make_atom(978)
#define am_bs_get_utf16 make_atom(979)
#define am_bs_validate_unicode make_atom(980)
#define am_bs_validate_unicode_retract make_atom(981)
#define am_emulate_fpe make_atom(982)
#define am_emasculate_binary make_atom(983)
#define am_is_divisible make_atom(984)
#define am_call_count_on make_atom(985)
#define am_call_count_off make_atom(986)
#define am_call_count_get make_atom(987)
#define am_call_count_clear make_atom(988)
#define am_trap_count_get make_atom(989)
#define am_trap_count_clear make_atom(990)
#define am_process_info_clear make_atom(991)
#define am_message_info make_atom(992)
#define am_message_info_clear make_atom(993)
#define am_message_sizes make_atom(994)
#define am_gc_info make_atom(995)
#define am_shared_gc_info make_atom(996)
#define am_incremental_gc_info make_atom(997)
#define am_gc_info_clear make_atom(998)
#define am_pause_times make_atom(999)
#define am_system_timer make_atom(1000)
#define am_system_timer_clear make_atom(1001)
#define am_send_timer make_atom(1002)
#define am_send_timer_clear make_atom(1003)
#define am_gc_timer make_atom(1004)
#define am_shared_gc_timer make_atom(1005)
#define am_gc_timer_clear make_atom(1006)
#define am_misc_timer make_atom(1007)
#define am_misc_timer_clear make_atom(1008)
#define am_get_hrvtime make_atom(1009)
#define am_stop_hrvtime make_atom(1010)
#define am_show_estack make_atom(1011)
#define am_show_heap make_atom(1012)
#define am_show_nstack make_atom(1013)
#define am_nstack_used_size make_atom(1014)
#define am_show_pcb make_atom(1015)
#define am_show_term make_atom(1016)
#define am_in_native make_atom(1017)
#define am_modeswitch_debug_on make_atom(1018)
#define am_modeswitch_debug_off make_atom(1019)
#define am_debug_native_called make_atom(1020)
#define am_llvm_fix_pinned_regs make_atom(1021)
#define am_arm make_atom(1022)
#define am_inc_stack_0 make_atom(1023)
#endif
# 1035 "armv6-portbld-freebsd11.0/erl_atom_table.h"
# 29 "beam/atom.h" 2

#define MAX_ATOM_CHARACTERS 255
#define MAX_ATOM_SZ_FROM_LATIN1 (2*MAX_ATOM_CHARACTERS)
#define MAX_ATOM_SZ_LIMIT (4*MAX_ATOM_CHARACTERS) /* theoretical byte limit */
#define ATOM_LIMIT (1024*1024)
#define MIN_ATOM_TABLE_SIZE 8192

#ifndef ARCH_32
/* Internal atom cache needs MAX_ATOM_TABLE_SIZE to be less than an
   unsigned 32 bit integer. See external.c(erts_encode_ext_dist_header_setup)
   for more details. */
#define MAX_ATOM_TABLE_SIZE ((MAX_ATOM_INDEX + 1 < (UWORD_CONSTANT(1) << 32)) ? MAX_ATOM_INDEX + 1 : (UWORD_CONSTANT(1) << 32))
#else
# 42 "beam/atom.h"
#define MAX_ATOM_TABLE_SIZE (MAX_ATOM_INDEX + 1)
#endif
# 44 "beam/atom.h"


/*
 * Atom entry.
 */
typedef struct atom {
    IndexSlot slot;  /* MUST BE LOCATED AT TOP OF STRUCT!!! */
    Sint16 len;      /* length of atom name (UTF-8 encoded) */
    Sint16 latin1_chars; /* 0-255 if atom can be encoded in latin1; otherwise, -1 */
    int ord0;        /* ordinal value of first 3 bytes + 7 bits */
    byte* name;      /* name of atom */
} Atom;

extern IndexTable erts_atom_table;

ERTS_GLB_INLINE Atom* atom_tab(Uint i);
ERTS_GLB_INLINE int erts_is_atom_utf8_bytes(byte *text, size_t len, Eterm term);
ERTS_GLB_INLINE int erts_is_atom_str(const char *str, Eterm term, int is_latin1);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Atom*
atom_tab(Uint i)
{
    return (Atom *) erts_index_lookup(&erts_atom_table, i);
}

ERTS_GLB_INLINE int erts_is_atom_utf8_bytes(byte *text, size_t len, Eterm term)
{
    Atom *a;
    if (!is_atom(term))
	return 0;
    a = atom_tab(atom_val(term));
    return (len == (size_t) a->len
	    && sys_memcmp((void *) a->name, (void *) text, len) == 0);
}

ERTS_GLB_INLINE int erts_is_atom_str(const char *str, Eterm term, int is_latin1)
{
    Atom *a;
    int i, len;
    const byte* aname;
    const byte* s = (const byte*) str;

    if (!is_atom(term))
	return 0;
    a = atom_tab(atom_val(term));
    len = a->len;
    aname = a->name;
    if (is_latin1) {
	for (i = 0; i < len; s++) {
	    if (aname[i] < 0x80) {
		if (aname[i] != *s || *s == '\0')
		    return 0;
		i++;
	    }
	    else {
		if (aname[i]   != (0xC0 | (*s >> 6)) || 
		    aname[i+1] != (0x80 | (*s & 0x3F))) {
		    return 0;
		}
		i += 2;
	    }
	}
    }
    else {
	for (i = 0; i < len; i++, s++)
	    if (aname[i] != *s || *s == '\0')
		return 0;
    }
    return *s == '\0';
}

#endif
# 117 "beam/atom.h"

typedef enum {
    ERTS_ATOM_ENC_7BIT_ASCII,
    ERTS_ATOM_ENC_LATIN1,
    ERTS_ATOM_ENC_UTF8
} ErtsAtomEncoding;

/*
 * Note, ERTS_IS_ATOM_STR() expects the first argument to be a
 * 7-bit ASCII string literal.
 */
#define ERTS_IS_ATOM_STR(LSTR, TERM) \
  (erts_is_atom_utf8_bytes((byte *) LSTR, sizeof(LSTR) - 1, (TERM)))
#define ERTS_DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)
#define ERTS_INIT_AM(S) AM_ ## S = am_atom_put(#S, sizeof(#S) - 1)

int atom_table_size(void);	/* number of elements */
int atom_table_sz(void);	/* table size in bytes, excluding stored objects */

Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */
Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc);
int atom_erase(byte*, int);
int atom_static_put(byte*, int);
void init_atom_table(void);
void atom_info(int, void *);
void dump_atoms(int, void *);
int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc);
void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used);
#endif
# 146 "beam/atom.h"

# 32 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "code_ix.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/global.h"
# 1 "beam/code_ix.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Description:
 *	This is the interface that facilitates changing the beam code
 *      (load,upgrade,delete) while allowing executing Erlang processes to
 *      access the code without any locks or other expensive memory barriers.
 *
 *      The basic idea is to maintain several "logical copies" of the code. These
 *      copies are identified by a global 'code index', an integer of 0, 1 or 2.
 *      The code index is used as argument to code access structures like
 *      export, module, beam_catches, beam_ranges.
 *
 *      The current 'active' code index is used to access the current running
 *      code. The 'staging' code index is used by the process that performs
 *      a code change operation. When a code change operation completes
 *      succesfully, the staging code index becomes the new active code index.
 *
 *      The third code index is not explicitly used. It can be thought of as
 *      the "previous active" or the "next staging" index. It is needed to make
 *      sure that we do not reuse a code index for staging until we are sure
 *      that no executing BIFs are still referencing it.
 *      We could get by with only two (0 and 1), but that would require that we
 *      must wait for all schedulers to re-schedule before each code change
 *      operation can start staging.
 *
 *      Note that the 'code index' is very loosely coupled to the concept of
 *      'current' and 'old' module versions. You can almost say that they are
 *      orthogonal to each other. Code index is an emulator global concept while
 *      'current' and 'old' is specific for each module.
 */

#ifndef __CODE_IX_H__
#define __CODE_IX_H__

#ifndef __SYS_H__
#  ifdef HAVE_CONFIG_H
#if 0 /* expanded by -frewrite-includes */
#    include "config.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/code_ix.h"
# 56 "beam/code_ix.h"
#  endif
# 57 "beam/code_ix.h"
#if 0 /* expanded by -frewrite-includes */
#  include "sys.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/code_ix.h"
# 58 "beam/code_ix.h"
#endif
# 59 "beam/code_ix.h"
struct process;


#define ERTS_NUM_CODE_IX 3
typedef unsigned ErtsCodeIndex;


/* Called once at emulator initialization.
 */
void erts_code_ix_init(void);

/* Return active code index.
 * Is guaranteed to be valid until the calling BIF returns.
 * To get a consistent view of the code, only one call to erts_active_code_ix()
 * should be made and the returned ix reused within the same BIF call.
 */
ERTS_GLB_INLINE
ErtsCodeIndex erts_active_code_ix(void);

/* Return staging code ix.
 * Only used by a process performing code loading/upgrading/deleting/purging.
 * Code write permission must be seized.
 */
ERTS_GLB_INLINE
ErtsCodeIndex erts_staging_code_ix(void);

/* Try seize exclusive code write permission. Needed for code staging.
 * Main process lock (only) must be held.
 * System thread progress must not be blocked.
 * Caller must not already hold the code write permission.
 * Caller is suspended and *must* yield if 0 is returned. 
 */
int erts_try_seize_code_write_permission(struct process* c_p);

/* Release code write permission.
 * Will resume any suspended waiters.
 */
void erts_release_code_write_permission(void);

/* Prepare the "staging area" to be a complete copy of the active code.
 * Code write permission must have been seized.
 * Must be followed by calls to either "end" and "commit" or "abort" before
 * code write permission can be released.
 */
void erts_start_staging_code_ix(void);

/* End the staging.
 * Preceded by "start" and must be followed by "commit".
 */
void erts_end_staging_code_ix(void);

/* Set staging code index as new active code index.
 * Preceded by "end".
 */
void erts_commit_staging_code_ix(void);

/* Abort the staging.
 * Preceded by "start".
 */
void erts_abort_staging_code_ix(void);

#ifdef ERTS_ENABLE_LOCK_CHECK
int erts_has_code_write_permission(void);
#endif
# 123 "beam/code_ix.h"



#if ERTS_GLB_INLINE_INCL_FUNC_DEF

extern erts_smp_atomic32_t the_active_code_index;
extern erts_smp_atomic32_t the_staging_code_index;

ERTS_GLB_INLINE ErtsCodeIndex erts_active_code_ix(void)
{
    return erts_smp_atomic32_read_nob(&the_active_code_index);
}
ERTS_GLB_INLINE ErtsCodeIndex erts_staging_code_ix(void)
{
    return erts_smp_atomic32_read_nob(&the_staging_code_index);
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 141 "beam/code_ix.h"

#endif /* !__CODE_IX_H__ */
# 143 "beam/code_ix.h"

# 33 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/global.h"
# 1 "beam/export.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __EXPORT_H__
#define __EXPORT_H__

#ifndef __SYS_H__
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/export.h"
# 26 "beam/export.h"
#endif
# 27 "beam/export.h"

#ifndef __INDEX_H__
#if 0 /* expanded by -frewrite-includes */
#include "index.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/export.h"
# 30 "beam/export.h"
#endif
# 31 "beam/export.h"

#if 0 /* expanded by -frewrite-includes */
#include "code_ix.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/export.h"
# 33 "beam/export.h"

/*
** Export entry
*/

typedef struct export
{
    void* addressv[ERTS_NUM_CODE_IX];  /* Pointer to code for function. */

    BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
    /*
     * code[0]: Tagged atom for module.
     * code[1]: Tagged atom for function.
     * code[2]: Arity (untagged integer).
     * code[3]: This entry is 0 unless the 'address' field points to it.
     *          Threaded code instruction to load function
     *		(em_call_error_handler), execute BIF (em_apply_bif),
     *		or a breakpoint instruction (op_i_generic_breakpoint).
     * code[4]: Function pointer to BIF function (for BIFs only),
     *		or pointer to threaded code if the module has an
     *		on_load function that has not been run yet, or pointer
     *          to code for function code[3] is a breakpont instruction.
     *		Otherwise: 0.
     */
    BeamInstr code[5];
} Export;


void init_export_table(void);
void export_info(int, void *);

ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a);
Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity);

Export* erts_export_get_or_make_stub(Eterm, Eterm, unsigned);

Export *export_list(int,ErtsCodeIndex);
int export_list_size(ErtsCodeIndex);
int export_table_sz(void);
int export_entries_sz(void);
Export *export_get(Export*);
void export_start_staging(void);
void export_end_staging(int commit);

extern erts_smp_mtx_t export_staging_lock;
#define export_staging_lock()	erts_smp_mtx_lock(&export_staging_lock)
#define export_staging_unlock()	erts_smp_mtx_unlock(&export_staging_lock)

#if 0 /* expanded by -frewrite-includes */
#include "beam_load.h" /* For em_* extern declarations */ 
#endif /* expanded by -frewrite-includes */
# 81 "beam/export.h"
# 1 "beam/beam_load.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1999-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef _BEAM_LOAD_H
#  define _BEAM_LOAD_H

#if 0 /* expanded by -frewrite-includes */
#include "beam_opcodes.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/beam_load.h"
# 1 "armv6-portbld-freebsd11.0/opt/smp/beam_opcodes.h" 1
/*
 *  Warning: Do not edit this file.
 *  Auto-generated by 'beam_makeops'.
 */

#ifndef __OPCODES_H__
#define __OPCODES_H__

#define BEAM_FORMAT_NUMBER 0
#define MAX_GENERIC_OPCODE 158
#define NUM_GENERIC_OPS 360
#define NUM_SPECIFIC_OPS 510

#ifdef ARCH_64
#  define BEAM_WIDE_MASK 0xFFFFUL
#  define BEAM_LOOSE_MASK 0x1FFFUL
#if HALFWORD_HEAP
#  define BEAM_TIGHT_MASK 0x1FFCUL
#else
# 20 "armv6-portbld-freebsd11.0/opt/smp/beam_opcodes.h"
#  define BEAM_TIGHT_MASK 0x1FF8UL
#endif
# 22 "armv6-portbld-freebsd11.0/opt/smp/beam_opcodes.h"
#  define BEAM_WIDE_SHIFT 32
#  define BEAM_LOOSE_SHIFT 16
#  define BEAM_TIGHT_SHIFT 16
#else
# 26 "armv6-portbld-freebsd11.0/opt/smp/beam_opcodes.h"
#  define BEAM_LOOSE_MASK 0xFFF
#  define BEAM_TIGHT_MASK 0xFFC
#  define BEAM_LOOSE_SHIFT 16
#  define BEAM_TIGHT_SHIFT 10
#endif
# 31 "armv6-portbld-freebsd11.0/opt/smp/beam_opcodes.h"

/*
 * The following operand types for generic instructions
 * occur in beam files.
 */

#define TAG_u 0
#define TAG_i 1
#define TAG_a 2
#define TAG_x 3
#define TAG_y 4
#define TAG_f 5
#define TAG_h 6
#define TAG_z 7

/*
 * The following operand types are only used in the loader.
 */

#define TAG_n 8
#define TAG_p 9
#define TAG_r 10
#define TAG_v 11
#define TAG_l 12
#define TAG_q 13
#define TAG_o 14

#define BEAM_NUM_TAGS 15

#define TOP_call_end 0
#define TOP_commit 1
#define TOP_end 2
#define TOP_fail 3
#define TOP_is_bif 4
#define TOP_is_func 5
#define TOP_is_not_bif 6
#define TOP_is_same_var 7
#define TOP_is_type 8
#define TOP_is_type_eq 9
#define TOP_new_instr 10
#define TOP_next_arg 11
#define TOP_next_instr 12
#define TOP_pred 13
#define TOP_rest_args 14
#define TOP_set_var_next_arg 15
#define TOP_store_rest_args 16
#define TOP_store_type 17
#define TOP_store_val 18
#define TOP_store_var_next_arg 19
#define TOP_try_me_else 20
#define TOP_try_me_else_fail 21
#define NUM_TOPS 22

#define TE_MAX_VARS 8

extern char tag_to_letter[];
extern Uint op_transform[];

#define op_allocate_tt 0
#define op_allocate_heap_tIt 1
#define op_allocate_heap_zero_tIt 2
#define op_allocate_init_tIy 3
#define op_allocate_zero_tt 4
#define op_apply_I 5
#define op_apply_bif 6
#define op_apply_last_IP 7
#define op_badarg_j 8
#define op_badmatch_r 9
#define op_badmatch_x 10
#define op_badmatch_y 11
#define op_bif1_fbsd 12
#define op_bif1_body_bsd 13
#define op_bs_context_to_binary_r 14
#define op_bs_context_to_binary_x 15
#define op_bs_context_to_binary_y 16
#define op_bs_init_writable 17
#define op_bs_put_string_II 18
#define op_bs_test_tail_imm2_frI 19
#define op_bs_test_tail_imm2_fxI 20
#define op_bs_test_unit_frI 21
#define op_bs_test_unit_fxI 22
#define op_bs_test_unit8_fr 23
#define op_bs_test_unit8_fx 24
#define op_bs_test_zero_tail2_fr 25
#define op_bs_test_zero_tail2_fx 26
#define op_call_bif_e 27
#define op_call_error_handler 28
#define op_call_nif 29
#define op_case_end_r 30
#define op_case_end_x 31
#define op_case_end_y 32
#define op_catch_yf 33
#define op_catch_end_y 34
#define op_continue_exit 35
#define op_deallocate_I 36
#define op_deallocate_return_Q 37
#define op_error_action_code 38
#define op_extract_next_element_x 39
#define op_extract_next_element_y 40
#define op_extract_next_element2_x 41
#define op_extract_next_element2_y 42
#define op_extract_next_element3_x 43
#define op_extract_next_element3_y 44
#define op_fclearerror 45
#define op_fconv_dl 46
#define op_fmove_ql 47
#define op_fmove_ld 48
#define op_fmove_dl 49
#define op_get_list_rrx 50
#define op_get_list_rry 51
#define op_get_list_rxr 52
#define op_get_list_rxx 53
#define op_get_list_rxy 54
#define op_get_list_ryr 55
#define op_get_list_ryx 56
#define op_get_list_ryy 57
#define op_get_list_xrx 58
#define op_get_list_xry 59
#define op_get_list_xxr 60
#define op_get_list_xxx 61
#define op_get_list_xxy 62
#define op_get_list_xyr 63
#define op_get_list_xyx 64
#define op_get_list_xyy 65
#define op_get_list_yrx 66
#define op_get_list_yry 67
#define op_get_list_yxr 68
#define op_get_list_yxx 69
#define op_get_list_yxy 70
#define op_get_list_yyr 71
#define op_get_list_yyx 72
#define op_get_list_yyy 73
#define op_hipe_call_count 74
#define op_hipe_trap_call 75
#define op_hipe_trap_call_closure 76
#define op_hipe_trap_resume 77
#define op_hipe_trap_return 78
#define op_hipe_trap_throw 79
#define op_i_apply 80
#define op_i_apply_fun 81
#define op_i_apply_fun_last_P 82
#define op_i_apply_fun_only 83
#define op_i_apply_last_P 84
#define op_i_apply_only 85
#define op_i_band_jId 86
#define op_i_band_jIxcd 87
#define op_i_bif2_fbd 88
#define op_i_bif2_body_bd 89
#define op_i_bor_jId 90
#define op_i_bs_add_jId 91
#define op_i_bs_append_jIIId 92
#define op_i_bs_get_binary2_frIsId 93
#define op_i_bs_get_binary2_fxIsId 94
#define op_i_bs_get_binary_all2_frIId 95
#define op_i_bs_get_binary_all2_fxIId 96
#define op_i_bs_get_binary_all_reuse_rfI 97
#define op_i_bs_get_binary_all_reuse_xfI 98
#define op_i_bs_get_binary_imm2_frIIId 99
#define op_i_bs_get_binary_imm2_fxIIId 100
#define op_i_bs_get_float2_frIsId 101
#define op_i_bs_get_float2_fxIsId 102
#define op_i_bs_get_integer_fIId 103
#define op_i_bs_get_integer_16_rfd 104
#define op_i_bs_get_integer_16_xfd 105
#define op_i_bs_get_integer_32_rfId 106
#define op_i_bs_get_integer_32_xfId 107
#define op_i_bs_get_integer_8_rfd 108
#define op_i_bs_get_integer_8_xfd 109
#define op_i_bs_get_integer_imm_rIIfId 110
#define op_i_bs_get_integer_imm_xIIfId 111
#define op_i_bs_get_integer_small_imm_rIfId 112
#define op_i_bs_get_integer_small_imm_xIfId 113
#define op_i_bs_get_utf16_rfId 114
#define op_i_bs_get_utf16_xfId 115
#define op_i_bs_get_utf8_rfd 116
#define op_i_bs_get_utf8_xfd 117
#define op_i_bs_init_IId 118
#define op_i_bs_init_bits_IId 119
#define op_i_bs_init_bits_fail_rjId 120
#define op_i_bs_init_bits_fail_xjId 121
#define op_i_bs_init_bits_fail_yjId 122
#define op_i_bs_init_bits_fail_heap_IjId 123
#define op_i_bs_init_bits_heap_IIId 124
#define op_i_bs_init_fail_rjId 125
#define op_i_bs_init_fail_xjId 126
#define op_i_bs_init_fail_yjId 127
#define op_i_bs_init_fail_heap_IjId 128
#define op_i_bs_init_heap_IIId 129
#define op_i_bs_init_heap_bin_IId 130
#define op_i_bs_init_heap_bin_heap_IIId 131
#define op_i_bs_match_string_rfII 132
#define op_i_bs_match_string_xfII 133
#define op_i_bs_private_append_jId 134
#define op_i_bs_put_utf16_jIs 135
#define op_i_bs_put_utf8_js 136
#define op_i_bs_restore2_rI 137
#define op_i_bs_restore2_xI 138
#define op_i_bs_save2_rI 139
#define op_i_bs_save2_xI 140
#define op_i_bs_skip_bits2_frxI 141
#define op_i_bs_skip_bits2_fryI 142
#define op_i_bs_skip_bits2_fxrI 143
#define op_i_bs_skip_bits2_fxxI 144
#define op_i_bs_skip_bits2_fxyI 145
#define op_i_bs_skip_bits_all2_frI 146
#define op_i_bs_skip_bits_all2_fxI 147
#define op_i_bs_skip_bits_imm2_frI 148
#define op_i_bs_skip_bits_imm2_fxI 149
#define op_i_bs_start_match2_rfIId 150
#define op_i_bs_start_match2_xfIId 151
#define op_i_bs_start_match2_yfIId 152
#define op_i_bs_utf16_size_sd 153
#define op_i_bs_utf8_size_sd 154
#define op_i_bs_validate_unicode_js 155
#define op_i_bs_validate_unicode_retract_j 156
#define op_i_bsl_jId 157
#define op_i_bsr_jId 158
#define op_i_bxor_jId 159
#define op_i_call_f 160
#define op_i_call_ext_e 161
#define op_i_call_ext_last_eP 162
#define op_i_call_ext_only_e 163
#define op_i_call_fun_I 164
#define op_i_call_fun_last_IP 165
#define op_i_call_last_fP 166
#define op_i_call_only_f 167
#define op_i_debug_breakpoint 168
#define op_i_element_rjsd 169
#define op_i_element_xjsd 170
#define op_i_element_yjsd 171
#define op_i_fadd_lll 172
#define op_i_fast_element_rjId 173
#define op_i_fast_element_xjId 174
#define op_i_fast_element_yjId 175
#define op_i_fcheckerror 176
#define op_i_fdiv_lll 177
#define op_i_fetch_rx 178
#define op_i_fetch_ry 179
#define op_i_fetch_xr 180
#define op_i_fetch_xx 181
#define op_i_fetch_xy 182
#define op_i_fetch_yr 183
#define op_i_fetch_yx 184
#define op_i_fetch_yy 185
#define op_i_fetch_rc 186
#define op_i_fetch_xc 187
#define op_i_fetch_yc 188
#define op_i_fetch_cr 189
#define op_i_fetch_cx 190
#define op_i_fetch_cy 191
#define op_i_fetch_cc 192
#define op_i_fetch_ss 193
#define op_i_fmul_lll 194
#define op_i_fnegate_ll 195
#define op_i_fsub_lll 196
#define op_i_func_info_IaaI 197
#define op_i_gc_bif1_jIsId 198
#define op_i_gc_bif2_jIId 199
#define op_i_gc_bif3_jIsId 200
#define op_i_generic_breakpoint 201
#define op_i_get_sd 202
#define op_i_get_map_element_frxr 203
#define op_i_get_map_element_frxx 204
#define op_i_get_map_element_frxy 205
#define op_i_get_map_element_fxxr 206
#define op_i_get_map_element_fxxx 207
#define op_i_get_map_element_fxxy 208
#define op_i_get_map_element_fyxr 209
#define op_i_get_map_element_fyxx 210
#define op_i_get_map_element_fyxy 211
#define op_i_get_map_element_hash_frcIr 212
#define op_i_get_map_element_hash_frcIx 213
#define op_i_get_map_element_hash_frcIy 214
#define op_i_get_map_element_hash_fxcIr 215
#define op_i_get_map_element_hash_fxcIx 216
#define op_i_get_map_element_hash_fxcIy 217
#define op_i_get_map_element_hash_fycIr 218
#define op_i_get_map_element_hash_fycIx 219
#define op_i_get_map_element_hash_fycIy 220
#define op_i_get_map_elements_fsI 221
#define op_i_get_tuple_element_rPr 222
#define op_i_get_tuple_element_rPx 223
#define op_i_get_tuple_element_rPy 224
#define op_i_get_tuple_element_xPr 225
#define op_i_get_tuple_element_xPx 226
#define op_i_get_tuple_element_xPy 227
#define op_i_get_tuple_element_yPr 228
#define op_i_get_tuple_element_yPx 229
#define op_i_get_tuple_element_yPy 230
#define op_i_hibernate 231
#define op_i_increment_rIId 232
#define op_i_increment_xIId 233
#define op_i_increment_yIId 234
#define op_i_int_bnot_jsId 235
#define op_i_int_div_jId 236
#define op_i_is_eq_f 237
#define op_i_is_eq_exact_f 238
#define op_i_is_eq_exact_immed_frc 239
#define op_i_is_eq_exact_immed_fxc 240
#define op_i_is_eq_exact_immed_fyc 241
#define op_i_is_eq_exact_literal_rfc 242
#define op_i_is_eq_exact_literal_xfc 243
#define op_i_is_eq_exact_literal_yfc 244
#define op_i_is_eq_exact_spec_frr 245
#define op_i_is_eq_exact_spec_frx 246
#define op_i_is_eq_exact_spec_fry 247
#define op_i_is_eq_exact_spec_fxx 248
#define op_i_is_eq_exact_spec_fxy 249
#define op_i_is_ge_f 250
#define op_i_is_ge_spec_fxx 251
#define op_i_is_ge_spec_fxc 252
#define op_i_is_ge_spec_fcx 253
#define op_i_is_ge_spec_fcc 254
#define op_i_is_lt_f 255
#define op_i_is_lt_spec_frr 256
#define op_i_is_lt_spec_frx 257
#define op_i_is_lt_spec_fxr 258
#define op_i_is_lt_spec_fxx 259
#define op_i_is_lt_spec_frc 260
#define op_i_is_lt_spec_fxc 261
#define op_i_is_lt_spec_fcr 262
#define op_i_is_lt_spec_fcx 263
#define op_i_is_lt_spec_fcc 264
#define op_i_is_ne_f 265
#define op_i_is_ne_exact_f 266
#define op_i_is_ne_exact_immed_frc 267
#define op_i_is_ne_exact_immed_fxc 268
#define op_i_is_ne_exact_immed_fyc 269
#define op_i_is_ne_exact_literal_rfc 270
#define op_i_is_ne_exact_literal_xfc 271
#define op_i_is_ne_exact_literal_yfc 272
#define op_i_jump_on_val_rfII 273
#define op_i_jump_on_val_xfII 274
#define op_i_jump_on_val_yfII 275
#define op_i_jump_on_val_zero_rfI 276
#define op_i_jump_on_val_zero_xfI 277
#define op_i_jump_on_val_zero_yfI 278
#define op_i_loop_rec_fr 279
#define op_i_m_div_jId 280
#define op_i_make_fun_It 281
#define op_i_minus_jId 282
#define op_i_minus_jIxxd 283
#define op_i_move_call_crf 284
#define op_i_move_call_ext_cre 285
#define op_i_move_call_ext_last_ePcr 286
#define op_i_move_call_ext_only_ecr 287
#define op_i_move_call_last_fPcr 288
#define op_i_move_call_only_fcr 289
#define op_i_new_bs_put_binary_jsIs 290
#define op_i_new_bs_put_binary_all_jsI 291
#define op_i_new_bs_put_binary_imm_jIs 292
#define op_i_new_bs_put_float_jsIs 293
#define op_i_new_bs_put_float_imm_jIIs 294
#define op_i_new_bs_put_integer_jsIs 295
#define op_i_new_bs_put_integer_imm_jIIs 296
#define op_i_plus_jId 297
#define op_i_plus_jIxxd 298
#define op_i_put_tuple_rI 299
#define op_i_put_tuple_xI 300
#define op_i_put_tuple_yI 301
#define op_i_recv_set 302
#define op_i_rem_jId 303
#define op_i_rem_jIxxd 304
#define op_i_return_time_trace 305
#define op_i_return_to_trace 306
#define op_i_select_tuple_arity_rfI 307
#define op_i_select_tuple_arity_xfI 308
#define op_i_select_tuple_arity_yfI 309
#define op_i_select_tuple_arity2_rfAAff 310
#define op_i_select_tuple_arity2_xfAAff 311
#define op_i_select_tuple_arity2_yfAAff 312
#define op_i_select_val2_rfccff 313
#define op_i_select_val2_xfccff 314
#define op_i_select_val2_yfccff 315
#define op_i_select_val_bins_rfI 316
#define op_i_select_val_bins_xfI 317
#define op_i_select_val_bins_yfI 318
#define op_i_select_val_lins_rfI 319
#define op_i_select_val_lins_xfI 320
#define op_i_select_val_lins_yfI 321
#define op_i_times_jId 322
#define op_i_trim_I 323
#define op_i_wait_error 324
#define op_i_wait_error_locked 325
#define op_i_wait_timeout_fI 326
#define op_i_wait_timeout_fs 327
#define op_i_wait_timeout_locked_fI 328
#define op_i_wait_timeout_locked_fs 329
#define op_i_yield 330
#define op_if_end 331
#define op_init_y 332
#define op_init2_yy 333
#define op_init3_yyy 334
#define op_int_code_end 335
#define op_is_atom_fr 336
#define op_is_atom_fx 337
#define op_is_atom_fy 338
#define op_is_binary_fr 339
#define op_is_binary_fx 340
#define op_is_binary_fy 341
#define op_is_bitstring_fr 342
#define op_is_bitstring_fx 343
#define op_is_bitstring_fy 344
#define op_is_boolean_fr 345
#define op_is_boolean_fx 346
#define op_is_boolean_fy 347
#define op_is_float_fr 348
#define op_is_float_fx 349
#define op_is_float_fy 350
#define op_is_function_fr 351
#define op_is_function_fx 352
#define op_is_function_fy 353
#define op_is_function2_fss 354
#define op_is_integer_fr 355
#define op_is_integer_fx 356
#define op_is_integer_fy 357
#define op_is_integer_allocate_frII 358
#define op_is_integer_allocate_fxII 359
#define op_is_list_fr 360
#define op_is_list_fx 361
#define op_is_list_fy 362
#define op_is_map_fr 363
#define op_is_map_fx 364
#define op_is_map_fy 365
#define op_is_nil_fr 366
#define op_is_nil_fx 367
#define op_is_nil_fy 368
#define op_is_non_empty_list_test_heap_frIt 369
#define op_is_nonempty_list_fr 370
#define op_is_nonempty_list_fx 371
#define op_is_nonempty_list_fy 372
#define op_is_nonempty_list_allocate_frIt 373
#define op_is_nonempty_list_allocate_fxIt 374
#define op_is_number_fr 375
#define op_is_number_fx 376
#define op_is_number_fy 377
#define op_is_pid_fr 378
#define op_is_pid_fx 379
#define op_is_pid_fy 380
#define op_is_port_fr 381
#define op_is_port_fx 382
#define op_is_port_fy 383
#define op_is_reference_fr 384
#define op_is_reference_fx 385
#define op_is_reference_fy 386
#define op_is_tuple_fr 387
#define op_is_tuple_fx 388
#define op_is_tuple_fy 389
#define op_is_tuple_of_arity_frA 390
#define op_is_tuple_of_arity_fxA 391
#define op_is_tuple_of_arity_fyA 392
#define op_jump_f 393
#define op_label_L 394
#define op_line_I 395
#define op_loop_rec_end_f 396
#define op_move_nr 397
#define op_move_nx 398
#define op_move_rx 399
#define op_move_ry 400
#define op_move_xr 401
#define op_move_xx 402
#define op_move_xy 403
#define op_move_yr 404
#define op_move_yx 405
#define op_move_yy 406
#define op_move_cr 407
#define op_move_cx 408
#define op_move2_rxyx 409
#define op_move2_ryxy 410
#define op_move2_xrxx 411
#define op_move2_xrxy 412
#define op_move2_xxxx 413
#define op_move2_xyxy 414
#define op_move2_yrxy 415
#define op_move2_yxrx 416
#define op_move2_yxyx 417
#define op_move3_xxxxxx 418
#define op_move3_xyxyxy 419
#define op_move3_yxyxyx 420
#define op_move_call_xrf 421
#define op_move_call_yrf 422
#define op_move_call_last_xrfQ 423
#define op_move_call_last_yrfQ 424
#define op_move_call_only_xrf 425
#define op_move_deallocate_return_nrQ 426
#define op_move_deallocate_return_xrQ 427
#define op_move_deallocate_return_yrQ 428
#define op_move_deallocate_return_crQ 429
#define op_move_jump_fn 430
#define op_move_jump_fx 431
#define op_move_jump_fy 432
#define op_move_jump_fc 433
#define op_move_return_nr 434
#define op_move_return_xr 435
#define op_move_return_cr 436
#define op_move_window3_xxxy 437
#define op_move_window4_xxxxy 438
#define op_move_window5_xxxxxy 439
#define op_move_x1_c 440
#define op_move_x2_c 441
#define op_new_map_dII 442
#define op_node_r 443
#define op_node_x 444
#define op_node_y 445
#define op_normal_exit 446
#define op_on_load 447
#define op_put_list_rnx 448
#define op_put_list_rxr 449
#define op_put_list_rxx 450
#define op_put_list_ryx 451
#define op_put_list_xnx 452
#define op_put_list_xrr 453
#define op_put_list_xrx 454
#define op_put_list_xxr 455
#define op_put_list_xxx 456
#define op_put_list_xyr 457
#define op_put_list_xyx 458
#define op_put_list_ynx 459
#define op_put_list_yrr 460
#define op_put_list_yrx 461
#define op_put_list_yxr 462
#define op_put_list_yxx 463
#define op_put_list_yyr 464
#define op_put_list_yyx 465
#define op_put_list_rcr 466
#define op_put_list_rcx 467
#define op_put_list_rcy 468
#define op_put_list_xcr 469
#define op_put_list_xcx 470
#define op_put_list_xcy 471
#define op_put_list_ycr 472
#define op_put_list_ycx 473
#define op_put_list_ycy 474
#define op_put_list_crr 475
#define op_put_list_crx 476
#define op_put_list_cry 477
#define op_put_list_cxr 478
#define op_put_list_cxx 479
#define op_put_list_cxy 480
#define op_put_list_cyr 481
#define op_put_list_cyx 482
#define op_put_list_cyy 483
#define op_put_list_ssd 484
#define op_raise_ss 485
#define op_recv_mark_f 486
#define op_remove_message 487
#define op_return 488
#define op_return_trace 489
#define op_self_r 490
#define op_self_x 491
#define op_self_y 492
#define op_send 493
#define op_set_tuple_element_sdP 494
#define op_system_limit_j 495
#define op_test_arity_frA 496
#define op_test_arity_fxA 497
#define op_test_arity_fyA 498
#define op_test_heap_It 499
#define op_test_heap_1_put_list_Iy 500
#define op_timeout 501
#define op_timeout_locked 502
#define op_try_case_end_s 503
#define op_try_end_y 504
#define op_update_map_assoc_jsdII 505
#define op_update_map_exact_jsdII 506
#define op_wait_f 507
#define op_wait_locked_f 508
#define op_wait_unlocked_f 509

#define NUMBER_OF_OPCODES 510
#define op_count_allocate_tt 510
#define op_count_allocate_heap_tIt 511
#define op_count_allocate_heap_zero_tIt 512
#define op_count_allocate_init_tIy 513
#define op_count_allocate_zero_tt 514
#define op_count_apply_I 515
#define op_count_apply_bif 516
#define op_count_apply_last_IP 517
#define op_count_badarg_j 518
#define op_count_badmatch_r 519
#define op_count_badmatch_x 520
#define op_count_badmatch_y 521
#define op_count_bif1_fbsd 522
#define op_count_bif1_body_bsd 523
#define op_count_bs_context_to_binary_r 524
#define op_count_bs_context_to_binary_x 525
#define op_count_bs_context_to_binary_y 526
#define op_count_bs_init_writable 527
#define op_count_bs_put_string_II 528
#define op_count_bs_test_tail_imm2_frI 529
#define op_count_bs_test_tail_imm2_fxI 530
#define op_count_bs_test_unit_frI 531
#define op_count_bs_test_unit_fxI 532
#define op_count_bs_test_unit8_fr 533
#define op_count_bs_test_unit8_fx 534
#define op_count_bs_test_zero_tail2_fr 535
#define op_count_bs_test_zero_tail2_fx 536
#define op_count_call_bif_e 537
#define op_count_call_error_handler 538
#define op_count_call_nif 539
#define op_count_case_end_r 540
#define op_count_case_end_x 541
#define op_count_case_end_y 542
#define op_count_catch_yf 543
#define op_count_catch_end_y 544
#define op_count_continue_exit 545
#define op_count_deallocate_I 546
#define op_count_deallocate_return_Q 547
#define op_count_error_action_code 548
#define op_count_extract_next_element_x 549
#define op_count_extract_next_element_y 550
#define op_count_extract_next_element2_x 551
#define op_count_extract_next_element2_y 552
#define op_count_extract_next_element3_x 553
#define op_count_extract_next_element3_y 554
#define op_count_fclearerror 555
#define op_count_fconv_dl 556
#define op_count_fmove_ql 557
#define op_count_fmove_ld 558
#define op_count_fmove_dl 559
#define op_count_get_list_rrx 560
#define op_count_get_list_rry 561
#define op_count_get_list_rxr 562
#define op_count_get_list_rxx 563
#define op_count_get_list_rxy 564
#define op_count_get_list_ryr 565
#define op_count_get_list_ryx 566
#define op_count_get_list_ryy 567
#define op_count_get_list_xrx 568
#define op_count_get_list_xry 569
#define op_count_get_list_xxr 570
#define op_count_get_list_xxx 571
#define op_count_get_list_xxy 572
#define op_count_get_list_xyr 573
#define op_count_get_list_xyx 574
#define op_count_get_list_xyy 575
#define op_count_get_list_yrx 576
#define op_count_get_list_yry 577
#define op_count_get_list_yxr 578
#define op_count_get_list_yxx 579
#define op_count_get_list_yxy 580
#define op_count_get_list_yyr 581
#define op_count_get_list_yyx 582
#define op_count_get_list_yyy 583
#define op_count_hipe_call_count 584
#define op_count_hipe_trap_call 585
#define op_count_hipe_trap_call_closure 586
#define op_count_hipe_trap_resume 587
#define op_count_hipe_trap_return 588
#define op_count_hipe_trap_throw 589
#define op_count_i_apply 590
#define op_count_i_apply_fun 591
#define op_count_i_apply_fun_last_P 592
#define op_count_i_apply_fun_only 593
#define op_count_i_apply_last_P 594
#define op_count_i_apply_only 595
#define op_count_i_band_jId 596
#define op_count_i_band_jIxcd 597
#define op_count_i_bif2_fbd 598
#define op_count_i_bif2_body_bd 599
#define op_count_i_bor_jId 600
#define op_count_i_bs_add_jId 601
#define op_count_i_bs_append_jIIId 602
#define op_count_i_bs_get_binary2_frIsId 603
#define op_count_i_bs_get_binary2_fxIsId 604
#define op_count_i_bs_get_binary_all2_frIId 605
#define op_count_i_bs_get_binary_all2_fxIId 606
#define op_count_i_bs_get_binary_all_reuse_rfI 607
#define op_count_i_bs_get_binary_all_reuse_xfI 608
#define op_count_i_bs_get_binary_imm2_frIIId 609
#define op_count_i_bs_get_binary_imm2_fxIIId 610
#define op_count_i_bs_get_float2_frIsId 611
#define op_count_i_bs_get_float2_fxIsId 612
#define op_count_i_bs_get_integer_fIId 613
#define op_count_i_bs_get_integer_16_rfd 614
#define op_count_i_bs_get_integer_16_xfd 615
#define op_count_i_bs_get_integer_32_rfId 616
#define op_count_i_bs_get_integer_32_xfId 617
#define op_count_i_bs_get_integer_8_rfd 618
#define op_count_i_bs_get_integer_8_xfd 619
#define op_count_i_bs_get_integer_imm_rIIfId 620
#define op_count_i_bs_get_integer_imm_xIIfId 621
#define op_count_i_bs_get_integer_small_imm_rIfId 622
#define op_count_i_bs_get_integer_small_imm_xIfId 623
#define op_count_i_bs_get_utf16_rfId 624
#define op_count_i_bs_get_utf16_xfId 625
#define op_count_i_bs_get_utf8_rfd 626
#define op_count_i_bs_get_utf8_xfd 627
#define op_count_i_bs_init_IId 628
#define op_count_i_bs_init_bits_IId 629
#define op_count_i_bs_init_bits_fail_rjId 630
#define op_count_i_bs_init_bits_fail_xjId 631
#define op_count_i_bs_init_bits_fail_yjId 632
#define op_count_i_bs_init_bits_fail_heap_IjId 633
#define op_count_i_bs_init_bits_heap_IIId 634
#define op_count_i_bs_init_fail_rjId 635
#define op_count_i_bs_init_fail_xjId 636
#define op_count_i_bs_init_fail_yjId 637
#define op_count_i_bs_init_fail_heap_IjId 638
#define op_count_i_bs_init_heap_IIId 639
#define op_count_i_bs_init_heap_bin_IId 640
#define op_count_i_bs_init_heap_bin_heap_IIId 641
#define op_count_i_bs_match_string_rfII 642
#define op_count_i_bs_match_string_xfII 643
#define op_count_i_bs_private_append_jId 644
#define op_count_i_bs_put_utf16_jIs 645
#define op_count_i_bs_put_utf8_js 646
#define op_count_i_bs_restore2_rI 647
#define op_count_i_bs_restore2_xI 648
#define op_count_i_bs_save2_rI 649
#define op_count_i_bs_save2_xI 650
#define op_count_i_bs_skip_bits2_frxI 651
#define op_count_i_bs_skip_bits2_fryI 652
#define op_count_i_bs_skip_bits2_fxrI 653
#define op_count_i_bs_skip_bits2_fxxI 654
#define op_count_i_bs_skip_bits2_fxyI 655
#define op_count_i_bs_skip_bits_all2_frI 656
#define op_count_i_bs_skip_bits_all2_fxI 657
#define op_count_i_bs_skip_bits_imm2_frI 658
#define op_count_i_bs_skip_bits_imm2_fxI 659
#define op_count_i_bs_start_match2_rfIId 660
#define op_count_i_bs_start_match2_xfIId 661
#define op_count_i_bs_start_match2_yfIId 662
#define op_count_i_bs_utf16_size_sd 663
#define op_count_i_bs_utf8_size_sd 664
#define op_count_i_bs_validate_unicode_js 665
#define op_count_i_bs_validate_unicode_retract_j 666
#define op_count_i_bsl_jId 667
#define op_count_i_bsr_jId 668
#define op_count_i_bxor_jId 669
#define op_count_i_call_f 670
#define op_count_i_call_ext_e 671
#define op_count_i_call_ext_last_eP 672
#define op_count_i_call_ext_only_e 673
#define op_count_i_call_fun_I 674
#define op_count_i_call_fun_last_IP 675
#define op_count_i_call_last_fP 676
#define op_count_i_call_only_f 677
#define op_count_i_debug_breakpoint 678
#define op_count_i_element_rjsd 679
#define op_count_i_element_xjsd 680
#define op_count_i_element_yjsd 681
#define op_count_i_fadd_lll 682
#define op_count_i_fast_element_rjId 683
#define op_count_i_fast_element_xjId 684
#define op_count_i_fast_element_yjId 685
#define op_count_i_fcheckerror 686
#define op_count_i_fdiv_lll 687
#define op_count_i_fetch_rx 688
#define op_count_i_fetch_ry 689
#define op_count_i_fetch_xr 690
#define op_count_i_fetch_xx 691
#define op_count_i_fetch_xy 692
#define op_count_i_fetch_yr 693
#define op_count_i_fetch_yx 694
#define op_count_i_fetch_yy 695
#define op_count_i_fetch_rc 696
#define op_count_i_fetch_xc 697
#define op_count_i_fetch_yc 698
#define op_count_i_fetch_cr 699
#define op_count_i_fetch_cx 700
#define op_count_i_fetch_cy 701
#define op_count_i_fetch_cc 702
#define op_count_i_fetch_ss 703
#define op_count_i_fmul_lll 704
#define op_count_i_fnegate_ll 705
#define op_count_i_fsub_lll 706
#define op_count_i_func_info_IaaI 707
#define op_count_i_gc_bif1_jIsId 708
#define op_count_i_gc_bif2_jIId 709
#define op_count_i_gc_bif3_jIsId 710
#define op_count_i_generic_breakpoint 711
#define op_count_i_get_sd 712
#define op_count_i_get_map_element_frxr 713
#define op_count_i_get_map_element_frxx 714
#define op_count_i_get_map_element_frxy 715
#define op_count_i_get_map_element_fxxr 716
#define op_count_i_get_map_element_fxxx 717
#define op_count_i_get_map_element_fxxy 718
#define op_count_i_get_map_element_fyxr 719
#define op_count_i_get_map_element_fyxx 720
#define op_count_i_get_map_element_fyxy 721
#define op_count_i_get_map_element_hash_frcIr 722
#define op_count_i_get_map_element_hash_frcIx 723
#define op_count_i_get_map_element_hash_frcIy 724
#define op_count_i_get_map_element_hash_fxcIr 725
#define op_count_i_get_map_element_hash_fxcIx 726
#define op_count_i_get_map_element_hash_fxcIy 727
#define op_count_i_get_map_element_hash_fycIr 728
#define op_count_i_get_map_element_hash_fycIx 729
#define op_count_i_get_map_element_hash_fycIy 730
#define op_count_i_get_map_elements_fsI 731
#define op_count_i_get_tuple_element_rPr 732
#define op_count_i_get_tuple_element_rPx 733
#define op_count_i_get_tuple_element_rPy 734
#define op_count_i_get_tuple_element_xPr 735
#define op_count_i_get_tuple_element_xPx 736
#define op_count_i_get_tuple_element_xPy 737
#define op_count_i_get_tuple_element_yPr 738
#define op_count_i_get_tuple_element_yPx 739
#define op_count_i_get_tuple_element_yPy 740
#define op_count_i_hibernate 741
#define op_count_i_increment_rIId 742
#define op_count_i_increment_xIId 743
#define op_count_i_increment_yIId 744
#define op_count_i_int_bnot_jsId 745
#define op_count_i_int_div_jId 746
#define op_count_i_is_eq_f 747
#define op_count_i_is_eq_exact_f 748
#define op_count_i_is_eq_exact_immed_frc 749
#define op_count_i_is_eq_exact_immed_fxc 750
#define op_count_i_is_eq_exact_immed_fyc 751
#define op_count_i_is_eq_exact_literal_rfc 752
#define op_count_i_is_eq_exact_literal_xfc 753
#define op_count_i_is_eq_exact_literal_yfc 754
#define op_count_i_is_eq_exact_spec_frr 755
#define op_count_i_is_eq_exact_spec_frx 756
#define op_count_i_is_eq_exact_spec_fry 757
#define op_count_i_is_eq_exact_spec_fxx 758
#define op_count_i_is_eq_exact_spec_fxy 759
#define op_count_i_is_ge_f 760
#define op_count_i_is_ge_spec_fxx 761
#define op_count_i_is_ge_spec_fxc 762
#define op_count_i_is_ge_spec_fcx 763
#define op_count_i_is_ge_spec_fcc 764
#define op_count_i_is_lt_f 765
#define op_count_i_is_lt_spec_frr 766
#define op_count_i_is_lt_spec_frx 767
#define op_count_i_is_lt_spec_fxr 768
#define op_count_i_is_lt_spec_fxx 769
#define op_count_i_is_lt_spec_frc 770
#define op_count_i_is_lt_spec_fxc 771
#define op_count_i_is_lt_spec_fcr 772
#define op_count_i_is_lt_spec_fcx 773
#define op_count_i_is_lt_spec_fcc 774
#define op_count_i_is_ne_f 775
#define op_count_i_is_ne_exact_f 776
#define op_count_i_is_ne_exact_immed_frc 777
#define op_count_i_is_ne_exact_immed_fxc 778
#define op_count_i_is_ne_exact_immed_fyc 779
#define op_count_i_is_ne_exact_literal_rfc 780
#define op_count_i_is_ne_exact_literal_xfc 781
#define op_count_i_is_ne_exact_literal_yfc 782
#define op_count_i_jump_on_val_rfII 783
#define op_count_i_jump_on_val_xfII 784
#define op_count_i_jump_on_val_yfII 785
#define op_count_i_jump_on_val_zero_rfI 786
#define op_count_i_jump_on_val_zero_xfI 787
#define op_count_i_jump_on_val_zero_yfI 788
#define op_count_i_loop_rec_fr 789
#define op_count_i_m_div_jId 790
#define op_count_i_make_fun_It 791
#define op_count_i_minus_jId 792
#define op_count_i_minus_jIxxd 793
#define op_count_i_move_call_crf 794
#define op_count_i_move_call_ext_cre 795
#define op_count_i_move_call_ext_last_ePcr 796
#define op_count_i_move_call_ext_only_ecr 797
#define op_count_i_move_call_last_fPcr 798
#define op_count_i_move_call_only_fcr 799
#define op_count_i_new_bs_put_binary_jsIs 800
#define op_count_i_new_bs_put_binary_all_jsI 801
#define op_count_i_new_bs_put_binary_imm_jIs 802
#define op_count_i_new_bs_put_float_jsIs 803
#define op_count_i_new_bs_put_float_imm_jIIs 804
#define op_count_i_new_bs_put_integer_jsIs 805
#define op_count_i_new_bs_put_integer_imm_jIIs 806
#define op_count_i_plus_jId 807
#define op_count_i_plus_jIxxd 808
#define op_count_i_put_tuple_rI 809
#define op_count_i_put_tuple_xI 810
#define op_count_i_put_tuple_yI 811
#define op_count_i_recv_set 812
#define op_count_i_rem_jId 813
#define op_count_i_rem_jIxxd 814
#define op_count_i_return_time_trace 815
#define op_count_i_return_to_trace 816
#define op_count_i_select_tuple_arity_rfI 817
#define op_count_i_select_tuple_arity_xfI 818
#define op_count_i_select_tuple_arity_yfI 819
#define op_count_i_select_tuple_arity2_rfAAff 820
#define op_count_i_select_tuple_arity2_xfAAff 821
#define op_count_i_select_tuple_arity2_yfAAff 822
#define op_count_i_select_val2_rfccff 823
#define op_count_i_select_val2_xfccff 824
#define op_count_i_select_val2_yfccff 825
#define op_count_i_select_val_bins_rfI 826
#define op_count_i_select_val_bins_xfI 827
#define op_count_i_select_val_bins_yfI 828
#define op_count_i_select_val_lins_rfI 829
#define op_count_i_select_val_lins_xfI 830
#define op_count_i_select_val_lins_yfI 831
#define op_count_i_times_jId 832
#define op_count_i_trim_I 833
#define op_count_i_wait_error 834
#define op_count_i_wait_error_locked 835
#define op_count_i_wait_timeout_fI 836
#define op_count_i_wait_timeout_fs 837
#define op_count_i_wait_timeout_locked_fI 838
#define op_count_i_wait_timeout_locked_fs 839
#define op_count_i_yield 840
#define op_count_if_end 841
#define op_count_init_y 842
#define op_count_init2_yy 843
#define op_count_init3_yyy 844
#define op_count_int_code_end 845
#define op_count_is_atom_fr 846
#define op_count_is_atom_fx 847
#define op_count_is_atom_fy 848
#define op_count_is_binary_fr 849
#define op_count_is_binary_fx 850
#define op_count_is_binary_fy 851
#define op_count_is_bitstring_fr 852
#define op_count_is_bitstring_fx 853
#define op_count_is_bitstring_fy 854
#define op_count_is_boolean_fr 855
#define op_count_is_boolean_fx 856
#define op_count_is_boolean_fy 857
#define op_count_is_float_fr 858
#define op_count_is_float_fx 859
#define op_count_is_float_fy 860
#define op_count_is_function_fr 861
#define op_count_is_function_fx 862
#define op_count_is_function_fy 863
#define op_count_is_function2_fss 864
#define op_count_is_integer_fr 865
#define op_count_is_integer_fx 866
#define op_count_is_integer_fy 867
#define op_count_is_integer_allocate_frII 868
#define op_count_is_integer_allocate_fxII 869
#define op_count_is_list_fr 870
#define op_count_is_list_fx 871
#define op_count_is_list_fy 872
#define op_count_is_map_fr 873
#define op_count_is_map_fx 874
#define op_count_is_map_fy 875
#define op_count_is_nil_fr 876
#define op_count_is_nil_fx 877
#define op_count_is_nil_fy 878
#define op_count_is_non_empty_list_test_heap_frIt 879
#define op_count_is_nonempty_list_fr 880
#define op_count_is_nonempty_list_fx 881
#define op_count_is_nonempty_list_fy 882
#define op_count_is_nonempty_list_allocate_frIt 883
#define op_count_is_nonempty_list_allocate_fxIt 884
#define op_count_is_number_fr 885
#define op_count_is_number_fx 886
#define op_count_is_number_fy 887
#define op_count_is_pid_fr 888
#define op_count_is_pid_fx 889
#define op_count_is_pid_fy 890
#define op_count_is_port_fr 891
#define op_count_is_port_fx 892
#define op_count_is_port_fy 893
#define op_count_is_reference_fr 894
#define op_count_is_reference_fx 895
#define op_count_is_reference_fy 896
#define op_count_is_tuple_fr 897
#define op_count_is_tuple_fx 898
#define op_count_is_tuple_fy 899
#define op_count_is_tuple_of_arity_frA 900
#define op_count_is_tuple_of_arity_fxA 901
#define op_count_is_tuple_of_arity_fyA 902
#define op_count_jump_f 903
#define op_count_label_L 904
#define op_count_line_I 905
#define op_count_loop_rec_end_f 906
#define op_count_move_nr 907
#define op_count_move_nx 908
#define op_count_move_rx 909
#define op_count_move_ry 910
#define op_count_move_xr 911
#define op_count_move_xx 912
#define op_count_move_xy 913
#define op_count_move_yr 914
#define op_count_move_yx 915
#define op_count_move_yy 916
#define op_count_move_cr 917
#define op_count_move_cx 918
#define op_count_move2_rxyx 919
#define op_count_move2_ryxy 920
#define op_count_move2_xrxx 921
#define op_count_move2_xrxy 922
#define op_count_move2_xxxx 923
#define op_count_move2_xyxy 924
#define op_count_move2_yrxy 925
#define op_count_move2_yxrx 926
#define op_count_move2_yxyx 927
#define op_count_move3_xxxxxx 928
#define op_count_move3_xyxyxy 929
#define op_count_move3_yxyxyx 930
#define op_count_move_call_xrf 931
#define op_count_move_call_yrf 932
#define op_count_move_call_last_xrfQ 933
#define op_count_move_call_last_yrfQ 934
#define op_count_move_call_only_xrf 935
#define op_count_move_deallocate_return_nrQ 936
#define op_count_move_deallocate_return_xrQ 937
#define op_count_move_deallocate_return_yrQ 938
#define op_count_move_deallocate_return_crQ 939
#define op_count_move_jump_fn 940
#define op_count_move_jump_fx 941
#define op_count_move_jump_fy 942
#define op_count_move_jump_fc 943
#define op_count_move_return_nr 944
#define op_count_move_return_xr 945
#define op_count_move_return_cr 946
#define op_count_move_window3_xxxy 947
#define op_count_move_window4_xxxxy 948
#define op_count_move_window5_xxxxxy 949
#define op_count_move_x1_c 950
#define op_count_move_x2_c 951
#define op_count_new_map_dII 952
#define op_count_node_r 953
#define op_count_node_x 954
#define op_count_node_y 955
#define op_count_normal_exit 956
#define op_count_on_load 957
#define op_count_put_list_rnx 958
#define op_count_put_list_rxr 959
#define op_count_put_list_rxx 960
#define op_count_put_list_ryx 961
#define op_count_put_list_xnx 962
#define op_count_put_list_xrr 963
#define op_count_put_list_xrx 964
#define op_count_put_list_xxr 965
#define op_count_put_list_xxx 966
#define op_count_put_list_xyr 967
#define op_count_put_list_xyx 968
#define op_count_put_list_ynx 969
#define op_count_put_list_yrr 970
#define op_count_put_list_yrx 971
#define op_count_put_list_yxr 972
#define op_count_put_list_yxx 973
#define op_count_put_list_yyr 974
#define op_count_put_list_yyx 975
#define op_count_put_list_rcr 976
#define op_count_put_list_rcx 977
#define op_count_put_list_rcy 978
#define op_count_put_list_xcr 979
#define op_count_put_list_xcx 980
#define op_count_put_list_xcy 981
#define op_count_put_list_ycr 982
#define op_count_put_list_ycx 983
#define op_count_put_list_ycy 984
#define op_count_put_list_crr 985
#define op_count_put_list_crx 986
#define op_count_put_list_cry 987
#define op_count_put_list_cxr 988
#define op_count_put_list_cxx 989
#define op_count_put_list_cxy 990
#define op_count_put_list_cyr 991
#define op_count_put_list_cyx 992
#define op_count_put_list_cyy 993
#define op_count_put_list_ssd 994
#define op_count_raise_ss 995
#define op_count_recv_mark_f 996
#define op_count_remove_message 997
#define op_count_return 998
#define op_count_return_trace 999
#define op_count_self_r 1000
#define op_count_self_x 1001
#define op_count_self_y 1002
#define op_count_send 1003
#define op_count_set_tuple_element_sdP 1004
#define op_count_system_limit_j 1005
#define op_count_test_arity_frA 1006
#define op_count_test_arity_fxA 1007
#define op_count_test_arity_fyA 1008
#define op_count_test_heap_It 1009
#define op_count_test_heap_1_put_list_Iy 1010
#define op_count_timeout 1011
#define op_count_timeout_locked 1012
#define op_count_try_case_end_s 1013
#define op_count_try_end_y 1014
#define op_count_update_map_assoc_jsdII 1015
#define op_count_update_map_exact_jsdII 1016
#define op_count_wait_f 1017
#define op_count_wait_locked_f 1018
#define op_count_wait_unlocked_f 1019

#define DEFINE_OPCODES \
&&lb_allocate_tt, \
&&lb_allocate_heap_tIt, \
&&lb_allocate_heap_zero_tIt, \
&&lb_allocate_init_tIy, \
&&lb_allocate_zero_tt, \
&&lb_apply_I, \
&&lb_apply_bif, \
&&lb_apply_last_IP, \
&&lb_badarg_j, \
&&lb_badmatch_r, \
&&lb_badmatch_x, \
&&lb_badmatch_y, \
&&lb_bif1_fbsd, \
&&lb_bif1_body_bsd, \
&&lb_bs_context_to_binary_r, \
&&lb_bs_context_to_binary_x, \
&&lb_bs_context_to_binary_y, \
&&lb_bs_init_writable, \
&&lb_bs_put_string_II, \
&&lb_bs_test_tail_imm2_frI, \
&&lb_bs_test_tail_imm2_fxI, \
&&lb_bs_test_unit_frI, \
&&lb_bs_test_unit_fxI, \
&&lb_bs_test_unit8_fr, \
&&lb_bs_test_unit8_fx, \
&&lb_bs_test_zero_tail2_fr, \
&&lb_bs_test_zero_tail2_fx, \
&&lb_call_bif_e, \
&&lb_call_error_handler, \
&&lb_call_nif, \
&&lb_case_end_r, \
&&lb_case_end_x, \
&&lb_case_end_y, \
&&lb_catch_yf, \
&&lb_catch_end_y, \
&&lb_continue_exit, \
&&lb_deallocate_I, \
&&lb_deallocate_return_Q, \
&&lb_error_action_code, \
&&lb_extract_next_element_x, \
&&lb_extract_next_element_y, \
&&lb_extract_next_element2_x, \
&&lb_extract_next_element2_y, \
&&lb_extract_next_element3_x, \
&&lb_extract_next_element3_y, \
&&lb_fclearerror, \
&&lb_fconv_dl, \
&&lb_fmove_ql, \
&&lb_fmove_ld, \
&&lb_fmove_dl, \
&&lb_get_list_rrx, \
&&lb_get_list_rry, \
&&lb_get_list_rxr, \
&&lb_get_list_rxx, \
&&lb_get_list_rxy, \
&&lb_get_list_ryr, \
&&lb_get_list_ryx, \
&&lb_get_list_ryy, \
&&lb_get_list_xrx, \
&&lb_get_list_xry, \
&&lb_get_list_xxr, \
&&lb_get_list_xxx, \
&&lb_get_list_xxy, \
&&lb_get_list_xyr, \
&&lb_get_list_xyx, \
&&lb_get_list_xyy, \
&&lb_get_list_yrx, \
&&lb_get_list_yry, \
&&lb_get_list_yxr, \
&&lb_get_list_yxx, \
&&lb_get_list_yxy, \
&&lb_get_list_yyr, \
&&lb_get_list_yyx, \
&&lb_get_list_yyy, \
&&lb_hipe_call_count, \
&&lb_hipe_trap_call, \
&&lb_hipe_trap_call_closure, \
&&lb_hipe_trap_resume, \
&&lb_hipe_trap_return, \
&&lb_hipe_trap_throw, \
&&lb_i_apply, \
&&lb_i_apply_fun, \
&&lb_i_apply_fun_last_P, \
&&lb_i_apply_fun_only, \
&&lb_i_apply_last_P, \
&&lb_i_apply_only, \
&&lb_i_band_jId, \
&&lb_i_band_jIxcd, \
&&lb_i_bif2_fbd, \
&&lb_i_bif2_body_bd, \
&&lb_i_bor_jId, \
&&lb_i_bs_add_jId, \
&&lb_i_bs_append_jIIId, \
&&lb_i_bs_get_binary2_frIsId, \
&&lb_i_bs_get_binary2_fxIsId, \
&&lb_i_bs_get_binary_all2_frIId, \
&&lb_i_bs_get_binary_all2_fxIId, \
&&lb_i_bs_get_binary_all_reuse_rfI, \
&&lb_i_bs_get_binary_all_reuse_xfI, \
&&lb_i_bs_get_binary_imm2_frIIId, \
&&lb_i_bs_get_binary_imm2_fxIIId, \
&&lb_i_bs_get_float2_frIsId, \
&&lb_i_bs_get_float2_fxIsId, \
&&lb_i_bs_get_integer_fIId, \
&&lb_i_bs_get_integer_16_rfd, \
&&lb_i_bs_get_integer_16_xfd, \
&&lb_i_bs_get_integer_32_rfId, \
&&lb_i_bs_get_integer_32_xfId, \
&&lb_i_bs_get_integer_8_rfd, \
&&lb_i_bs_get_integer_8_xfd, \
&&lb_i_bs_get_integer_imm_rIIfId, \
&&lb_i_bs_get_integer_imm_xIIfId, \
&&lb_i_bs_get_integer_small_imm_rIfId, \
&&lb_i_bs_get_integer_small_imm_xIfId, \
&&lb_i_bs_get_utf16_rfId, \
&&lb_i_bs_get_utf16_xfId, \
&&lb_i_bs_get_utf8_rfd, \
&&lb_i_bs_get_utf8_xfd, \
&&lb_i_bs_init_IId, \
&&lb_i_bs_init_bits_IId, \
&&lb_i_bs_init_bits_fail_rjId, \
&&lb_i_bs_init_bits_fail_xjId, \
&&lb_i_bs_init_bits_fail_yjId, \
&&lb_i_bs_init_bits_fail_heap_IjId, \
&&lb_i_bs_init_bits_heap_IIId, \
&&lb_i_bs_init_fail_rjId, \
&&lb_i_bs_init_fail_xjId, \
&&lb_i_bs_init_fail_yjId, \
&&lb_i_bs_init_fail_heap_IjId, \
&&lb_i_bs_init_heap_IIId, \
&&lb_i_bs_init_heap_bin_IId, \
&&lb_i_bs_init_heap_bin_heap_IIId, \
&&lb_i_bs_match_string_rfII, \
&&lb_i_bs_match_string_xfII, \
&&lb_i_bs_private_append_jId, \
&&lb_i_bs_put_utf16_jIs, \
&&lb_i_bs_put_utf8_js, \
&&lb_i_bs_restore2_rI, \
&&lb_i_bs_restore2_xI, \
&&lb_i_bs_save2_rI, \
&&lb_i_bs_save2_xI, \
&&lb_i_bs_skip_bits2_frxI, \
&&lb_i_bs_skip_bits2_fryI, \
&&lb_i_bs_skip_bits2_fxrI, \
&&lb_i_bs_skip_bits2_fxxI, \
&&lb_i_bs_skip_bits2_fxyI, \
&&lb_i_bs_skip_bits_all2_frI, \
&&lb_i_bs_skip_bits_all2_fxI, \
&&lb_i_bs_skip_bits_imm2_frI, \
&&lb_i_bs_skip_bits_imm2_fxI, \
&&lb_i_bs_start_match2_rfIId, \
&&lb_i_bs_start_match2_xfIId, \
&&lb_i_bs_start_match2_yfIId, \
&&lb_i_bs_utf16_size_sd, \
&&lb_i_bs_utf8_size_sd, \
&&lb_i_bs_validate_unicode_js, \
&&lb_i_bs_validate_unicode_retract_j, \
&&lb_i_bsl_jId, \
&&lb_i_bsr_jId, \
&&lb_i_bxor_jId, \
&&lb_i_call_f, \
&&lb_i_call_ext_e, \
&&lb_i_call_ext_last_eP, \
&&lb_i_call_ext_only_e, \
&&lb_i_call_fun_I, \
&&lb_i_call_fun_last_IP, \
&&lb_i_call_last_fP, \
&&lb_i_call_only_f, \
&&lb_i_debug_breakpoint, \
&&lb_i_element_rjsd, \
&&lb_i_element_xjsd, \
&&lb_i_element_yjsd, \
&&lb_i_fadd_lll, \
&&lb_i_fast_element_rjId, \
&&lb_i_fast_element_xjId, \
&&lb_i_fast_element_yjId, \
&&lb_i_fcheckerror, \
&&lb_i_fdiv_lll, \
&&lb_i_fetch_rx, \
&&lb_i_fetch_ry, \
&&lb_i_fetch_xr, \
&&lb_i_fetch_xx, \
&&lb_i_fetch_xy, \
&&lb_i_fetch_yr, \
&&lb_i_fetch_yx, \
&&lb_i_fetch_yy, \
&&lb_i_fetch_rc, \
&&lb_i_fetch_xc, \
&&lb_i_fetch_yc, \
&&lb_i_fetch_cr, \
&&lb_i_fetch_cx, \
&&lb_i_fetch_cy, \
&&lb_i_fetch_cc, \
&&lb_i_fetch_ss, \
&&lb_i_fmul_lll, \
&&lb_i_fnegate_ll, \
&&lb_i_fsub_lll, \
&&lb_i_func_info_IaaI, \
&&lb_i_gc_bif1_jIsId, \
&&lb_i_gc_bif2_jIId, \
&&lb_i_gc_bif3_jIsId, \
&&lb_i_generic_breakpoint, \
&&lb_i_get_sd, \
&&lb_i_get_map_element_frxr, \
&&lb_i_get_map_element_frxx, \
&&lb_i_get_map_element_frxy, \
&&lb_i_get_map_element_fxxr, \
&&lb_i_get_map_element_fxxx, \
&&lb_i_get_map_element_fxxy, \
&&lb_i_get_map_element_fyxr, \
&&lb_i_get_map_element_fyxx, \
&&lb_i_get_map_element_fyxy, \
&&lb_i_get_map_element_hash_frcIr, \
&&lb_i_get_map_element_hash_frcIx, \
&&lb_i_get_map_element_hash_frcIy, \
&&lb_i_get_map_element_hash_fxcIr, \
&&lb_i_get_map_element_hash_fxcIx, \
&&lb_i_get_map_element_hash_fxcIy, \
&&lb_i_get_map_element_hash_fycIr, \
&&lb_i_get_map_element_hash_fycIx, \
&&lb_i_get_map_element_hash_fycIy, \
&&lb_i_get_map_elements_fsI, \
&&lb_i_get_tuple_element_rPr, \
&&lb_i_get_tuple_element_rPx, \
&&lb_i_get_tuple_element_rPy, \
&&lb_i_get_tuple_element_xPr, \
&&lb_i_get_tuple_element_xPx, \
&&lb_i_get_tuple_element_xPy, \
&&lb_i_get_tuple_element_yPr, \
&&lb_i_get_tuple_element_yPx, \
&&lb_i_get_tuple_element_yPy, \
&&lb_i_hibernate, \
&&lb_i_increment_rIId, \
&&lb_i_increment_xIId, \
&&lb_i_increment_yIId, \
&&lb_i_int_bnot_jsId, \
&&lb_i_int_div_jId, \
&&lb_i_is_eq_f, \
&&lb_i_is_eq_exact_f, \
&&lb_i_is_eq_exact_immed_frc, \
&&lb_i_is_eq_exact_immed_fxc, \
&&lb_i_is_eq_exact_immed_fyc, \
&&lb_i_is_eq_exact_literal_rfc, \
&&lb_i_is_eq_exact_literal_xfc, \
&&lb_i_is_eq_exact_literal_yfc, \
&&lb_i_is_eq_exact_spec_frr, \
&&lb_i_is_eq_exact_spec_frx, \
&&lb_i_is_eq_exact_spec_fry, \
&&lb_i_is_eq_exact_spec_fxx, \
&&lb_i_is_eq_exact_spec_fxy, \
&&lb_i_is_ge_f, \
&&lb_i_is_ge_spec_fxx, \
&&lb_i_is_ge_spec_fxc, \
&&lb_i_is_ge_spec_fcx, \
&&lb_i_is_ge_spec_fcc, \
&&lb_i_is_lt_f, \
&&lb_i_is_lt_spec_frr, \
&&lb_i_is_lt_spec_frx, \
&&lb_i_is_lt_spec_fxr, \
&&lb_i_is_lt_spec_fxx, \
&&lb_i_is_lt_spec_frc, \
&&lb_i_is_lt_spec_fxc, \
&&lb_i_is_lt_spec_fcr, \
&&lb_i_is_lt_spec_fcx, \
&&lb_i_is_lt_spec_fcc, \
&&lb_i_is_ne_f, \
&&lb_i_is_ne_exact_f, \
&&lb_i_is_ne_exact_immed_frc, \
&&lb_i_is_ne_exact_immed_fxc, \
&&lb_i_is_ne_exact_immed_fyc, \
&&lb_i_is_ne_exact_literal_rfc, \
&&lb_i_is_ne_exact_literal_xfc, \
&&lb_i_is_ne_exact_literal_yfc, \
&&lb_i_jump_on_val_rfII, \
&&lb_i_jump_on_val_xfII, \
&&lb_i_jump_on_val_yfII, \
&&lb_i_jump_on_val_zero_rfI, \
&&lb_i_jump_on_val_zero_xfI, \
&&lb_i_jump_on_val_zero_yfI, \
&&lb_i_loop_rec_fr, \
&&lb_i_m_div_jId, \
&&lb_i_make_fun_It, \
&&lb_i_minus_jId, \
&&lb_i_minus_jIxxd, \
&&lb_i_move_call_crf, \
&&lb_i_move_call_ext_cre, \
&&lb_i_move_call_ext_last_ePcr, \
&&lb_i_move_call_ext_only_ecr, \
&&lb_i_move_call_last_fPcr, \
&&lb_i_move_call_only_fcr, \
&&lb_i_new_bs_put_binary_jsIs, \
&&lb_i_new_bs_put_binary_all_jsI, \
&&lb_i_new_bs_put_binary_imm_jIs, \
&&lb_i_new_bs_put_float_jsIs, \
&&lb_i_new_bs_put_float_imm_jIIs, \
&&lb_i_new_bs_put_integer_jsIs, \
&&lb_i_new_bs_put_integer_imm_jIIs, \
&&lb_i_plus_jId, \
&&lb_i_plus_jIxxd, \
&&lb_i_put_tuple_rI, \
&&lb_i_put_tuple_xI, \
&&lb_i_put_tuple_yI, \
&&lb_i_recv_set, \
&&lb_i_rem_jId, \
&&lb_i_rem_jIxxd, \
&&lb_i_return_time_trace, \
&&lb_i_return_to_trace, \
&&lb_i_select_tuple_arity_rfI, \
&&lb_i_select_tuple_arity_xfI, \
&&lb_i_select_tuple_arity_yfI, \
&&lb_i_select_tuple_arity2_rfAAff, \
&&lb_i_select_tuple_arity2_xfAAff, \
&&lb_i_select_tuple_arity2_yfAAff, \
&&lb_i_select_val2_rfccff, \
&&lb_i_select_val2_xfccff, \
&&lb_i_select_val2_yfccff, \
&&lb_i_select_val_bins_rfI, \
&&lb_i_select_val_bins_xfI, \
&&lb_i_select_val_bins_yfI, \
&&lb_i_select_val_lins_rfI, \
&&lb_i_select_val_lins_xfI, \
&&lb_i_select_val_lins_yfI, \
&&lb_i_times_jId, \
&&lb_i_trim_I, \
&&lb_i_wait_error, \
&&lb_i_wait_error_locked, \
&&lb_i_wait_timeout_fI, \
&&lb_i_wait_timeout_fs, \
&&lb_i_wait_timeout_locked_fI, \
&&lb_i_wait_timeout_locked_fs, \
&&lb_i_yield, \
&&lb_if_end, \
&&lb_init_y, \
&&lb_init2_yy, \
&&lb_init3_yyy, \
&&lb_int_code_end, \
&&lb_is_atom_fr, \
&&lb_is_atom_fx, \
&&lb_is_atom_fy, \
&&lb_is_binary_fr, \
&&lb_is_binary_fx, \
&&lb_is_binary_fy, \
&&lb_is_bitstring_fr, \
&&lb_is_bitstring_fx, \
&&lb_is_bitstring_fy, \
&&lb_is_boolean_fr, \
&&lb_is_boolean_fx, \
&&lb_is_boolean_fy, \
&&lb_is_float_fr, \
&&lb_is_float_fx, \
&&lb_is_float_fy, \
&&lb_is_function_fr, \
&&lb_is_function_fx, \
&&lb_is_function_fy, \
&&lb_is_function2_fss, \
&&lb_is_integer_fr, \
&&lb_is_integer_fx, \
&&lb_is_integer_fy, \
&&lb_is_integer_allocate_frII, \
&&lb_is_integer_allocate_fxII, \
&&lb_is_list_fr, \
&&lb_is_list_fx, \
&&lb_is_list_fy, \
&&lb_is_map_fr, \
&&lb_is_map_fx, \
&&lb_is_map_fy, \
&&lb_is_nil_fr, \
&&lb_is_nil_fx, \
&&lb_is_nil_fy, \
&&lb_is_non_empty_list_test_heap_frIt, \
&&lb_is_nonempty_list_fr, \
&&lb_is_nonempty_list_fx, \
&&lb_is_nonempty_list_fy, \
&&lb_is_nonempty_list_allocate_frIt, \
&&lb_is_nonempty_list_allocate_fxIt, \
&&lb_is_number_fr, \
&&lb_is_number_fx, \
&&lb_is_number_fy, \
&&lb_is_pid_fr, \
&&lb_is_pid_fx, \
&&lb_is_pid_fy, \
&&lb_is_port_fr, \
&&lb_is_port_fx, \
&&lb_is_port_fy, \
&&lb_is_reference_fr, \
&&lb_is_reference_fx, \
&&lb_is_reference_fy, \
&&lb_is_tuple_fr, \
&&lb_is_tuple_fx, \
&&lb_is_tuple_fy, \
&&lb_is_tuple_of_arity_frA, \
&&lb_is_tuple_of_arity_fxA, \
&&lb_is_tuple_of_arity_fyA, \
&&lb_jump_f, \
&&lb_label_L, \
&&lb_line_I, \
&&lb_loop_rec_end_f, \
&&lb_move_nr, \
&&lb_move_nx, \
&&lb_move_rx, \
&&lb_move_ry, \
&&lb_move_xr, \
&&lb_move_xx, \
&&lb_move_xy, \
&&lb_move_yr, \
&&lb_move_yx, \
&&lb_move_yy, \
&&lb_move_cr, \
&&lb_move_cx, \
&&lb_move2_rxyx, \
&&lb_move2_ryxy, \
&&lb_move2_xrxx, \
&&lb_move2_xrxy, \
&&lb_move2_xxxx, \
&&lb_move2_xyxy, \
&&lb_move2_yrxy, \
&&lb_move2_yxrx, \
&&lb_move2_yxyx, \
&&lb_move3_xxxxxx, \
&&lb_move3_xyxyxy, \
&&lb_move3_yxyxyx, \
&&lb_move_call_xrf, \
&&lb_move_call_yrf, \
&&lb_move_call_last_xrfQ, \
&&lb_move_call_last_yrfQ, \
&&lb_move_call_only_xrf, \
&&lb_move_deallocate_return_nrQ, \
&&lb_move_deallocate_return_xrQ, \
&&lb_move_deallocate_return_yrQ, \
&&lb_move_deallocate_return_crQ, \
&&lb_move_jump_fn, \
&&lb_move_jump_fx, \
&&lb_move_jump_fy, \
&&lb_move_jump_fc, \
&&lb_move_return_nr, \
&&lb_move_return_xr, \
&&lb_move_return_cr, \
&&lb_move_window3_xxxy, \
&&lb_move_window4_xxxxy, \
&&lb_move_window5_xxxxxy, \
&&lb_move_x1_c, \
&&lb_move_x2_c, \
&&lb_new_map_dII, \
&&lb_node_r, \
&&lb_node_x, \
&&lb_node_y, \
&&lb_normal_exit, \
&&lb_on_load, \
&&lb_put_list_rnx, \
&&lb_put_list_rxr, \
&&lb_put_list_rxx, \
&&lb_put_list_ryx, \
&&lb_put_list_xnx, \
&&lb_put_list_xrr, \
&&lb_put_list_xrx, \
&&lb_put_list_xxr, \
&&lb_put_list_xxx, \
&&lb_put_list_xyr, \
&&lb_put_list_xyx, \
&&lb_put_list_ynx, \
&&lb_put_list_yrr, \
&&lb_put_list_yrx, \
&&lb_put_list_yxr, \
&&lb_put_list_yxx, \
&&lb_put_list_yyr, \
&&lb_put_list_yyx, \
&&lb_put_list_rcr, \
&&lb_put_list_rcx, \
&&lb_put_list_rcy, \
&&lb_put_list_xcr, \
&&lb_put_list_xcx, \
&&lb_put_list_xcy, \
&&lb_put_list_ycr, \
&&lb_put_list_ycx, \
&&lb_put_list_ycy, \
&&lb_put_list_crr, \
&&lb_put_list_crx, \
&&lb_put_list_cry, \
&&lb_put_list_cxr, \
&&lb_put_list_cxx, \
&&lb_put_list_cxy, \
&&lb_put_list_cyr, \
&&lb_put_list_cyx, \
&&lb_put_list_cyy, \
&&lb_put_list_ssd, \
&&lb_raise_ss, \
&&lb_recv_mark_f, \
&&lb_remove_message, \
&&lb_return, \
&&lb_return_trace, \
&&lb_self_r, \
&&lb_self_x, \
&&lb_self_y, \
&&lb_send, \
&&lb_set_tuple_element_sdP, \
&&lb_system_limit_j, \
&&lb_test_arity_frA, \
&&lb_test_arity_fxA, \
&&lb_test_arity_fyA, \
&&lb_test_heap_It, \
&&lb_test_heap_1_put_list_Iy, \
&&lb_timeout, \
&&lb_timeout_locked, \
&&lb_try_case_end_s, \
&&lb_try_end_y, \
&&lb_update_map_assoc_jsdII, \
&&lb_update_map_exact_jsdII, \
&&lb_wait_f, \
&&lb_wait_locked_f, \
&&lb_wait_unlocked_f,

#define DEFINE_COUNTING_OPCODES \
&&lb_count_allocate_tt, \
&&lb_count_allocate_heap_tIt, \
&&lb_count_allocate_heap_zero_tIt, \
&&lb_count_allocate_init_tIy, \
&&lb_count_allocate_zero_tt, \
&&lb_count_apply_I, \
&&lb_count_apply_bif, \
&&lb_count_apply_last_IP, \
&&lb_count_badarg_j, \
&&lb_count_badmatch_r, \
&&lb_count_badmatch_x, \
&&lb_count_badmatch_y, \
&&lb_count_bif1_fbsd, \
&&lb_count_bif1_body_bsd, \
&&lb_count_bs_context_to_binary_r, \
&&lb_count_bs_context_to_binary_x, \
&&lb_count_bs_context_to_binary_y, \
&&lb_count_bs_init_writable, \
&&lb_count_bs_put_string_II, \
&&lb_count_bs_test_tail_imm2_frI, \
&&lb_count_bs_test_tail_imm2_fxI, \
&&lb_count_bs_test_unit_frI, \
&&lb_count_bs_test_unit_fxI, \
&&lb_count_bs_test_unit8_fr, \
&&lb_count_bs_test_unit8_fx, \
&&lb_count_bs_test_zero_tail2_fr, \
&&lb_count_bs_test_zero_tail2_fx, \
&&lb_count_call_bif_e, \
&&lb_count_call_error_handler, \
&&lb_count_call_nif, \
&&lb_count_case_end_r, \
&&lb_count_case_end_x, \
&&lb_count_case_end_y, \
&&lb_count_catch_yf, \
&&lb_count_catch_end_y, \
&&lb_count_continue_exit, \
&&lb_count_deallocate_I, \
&&lb_count_deallocate_return_Q, \
&&lb_count_error_action_code, \
&&lb_count_extract_next_element_x, \
&&lb_count_extract_next_element_y, \
&&lb_count_extract_next_element2_x, \
&&lb_count_extract_next_element2_y, \
&&lb_count_extract_next_element3_x, \
&&lb_count_extract_next_element3_y, \
&&lb_count_fclearerror, \
&&lb_count_fconv_dl, \
&&lb_count_fmove_ql, \
&&lb_count_fmove_ld, \
&&lb_count_fmove_dl, \
&&lb_count_get_list_rrx, \
&&lb_count_get_list_rry, \
&&lb_count_get_list_rxr, \
&&lb_count_get_list_rxx, \
&&lb_count_get_list_rxy, \
&&lb_count_get_list_ryr, \
&&lb_count_get_list_ryx, \
&&lb_count_get_list_ryy, \
&&lb_count_get_list_xrx, \
&&lb_count_get_list_xry, \
&&lb_count_get_list_xxr, \
&&lb_count_get_list_xxx, \
&&lb_count_get_list_xxy, \
&&lb_count_get_list_xyr, \
&&lb_count_get_list_xyx, \
&&lb_count_get_list_xyy, \
&&lb_count_get_list_yrx, \
&&lb_count_get_list_yry, \
&&lb_count_get_list_yxr, \
&&lb_count_get_list_yxx, \
&&lb_count_get_list_yxy, \
&&lb_count_get_list_yyr, \
&&lb_count_get_list_yyx, \
&&lb_count_get_list_yyy, \
&&lb_count_hipe_call_count, \
&&lb_count_hipe_trap_call, \
&&lb_count_hipe_trap_call_closure, \
&&lb_count_hipe_trap_resume, \
&&lb_count_hipe_trap_return, \
&&lb_count_hipe_trap_throw, \
&&lb_count_i_apply, \
&&lb_count_i_apply_fun, \
&&lb_count_i_apply_fun_last_P, \
&&lb_count_i_apply_fun_only, \
&&lb_count_i_apply_last_P, \
&&lb_count_i_apply_only, \
&&lb_count_i_band_jId, \
&&lb_count_i_band_jIxcd, \
&&lb_count_i_bif2_fbd, \
&&lb_count_i_bif2_body_bd, \
&&lb_count_i_bor_jId, \
&&lb_count_i_bs_add_jId, \
&&lb_count_i_bs_append_jIIId, \
&&lb_count_i_bs_get_binary2_frIsId, \
&&lb_count_i_bs_get_binary2_fxIsId, \
&&lb_count_i_bs_get_binary_all2_frIId, \
&&lb_count_i_bs_get_binary_all2_fxIId, \
&&lb_count_i_bs_get_binary_all_reuse_rfI, \
&&lb_count_i_bs_get_binary_all_reuse_xfI, \
&&lb_count_i_bs_get_binary_imm2_frIIId, \
&&lb_count_i_bs_get_binary_imm2_fxIIId, \
&&lb_count_i_bs_get_float2_frIsId, \
&&lb_count_i_bs_get_float2_fxIsId, \
&&lb_count_i_bs_get_integer_fIId, \
&&lb_count_i_bs_get_integer_16_rfd, \
&&lb_count_i_bs_get_integer_16_xfd, \
&&lb_count_i_bs_get_integer_32_rfId, \
&&lb_count_i_bs_get_integer_32_xfId, \
&&lb_count_i_bs_get_integer_8_rfd, \
&&lb_count_i_bs_get_integer_8_xfd, \
&&lb_count_i_bs_get_integer_imm_rIIfId, \
&&lb_count_i_bs_get_integer_imm_xIIfId, \
&&lb_count_i_bs_get_integer_small_imm_rIfId, \
&&lb_count_i_bs_get_integer_small_imm_xIfId, \
&&lb_count_i_bs_get_utf16_rfId, \
&&lb_count_i_bs_get_utf16_xfId, \
&&lb_count_i_bs_get_utf8_rfd, \
&&lb_count_i_bs_get_utf8_xfd, \
&&lb_count_i_bs_init_IId, \
&&lb_count_i_bs_init_bits_IId, \
&&lb_count_i_bs_init_bits_fail_rjId, \
&&lb_count_i_bs_init_bits_fail_xjId, \
&&lb_count_i_bs_init_bits_fail_yjId, \
&&lb_count_i_bs_init_bits_fail_heap_IjId, \
&&lb_count_i_bs_init_bits_heap_IIId, \
&&lb_count_i_bs_init_fail_rjId, \
&&lb_count_i_bs_init_fail_xjId, \
&&lb_count_i_bs_init_fail_yjId, \
&&lb_count_i_bs_init_fail_heap_IjId, \
&&lb_count_i_bs_init_heap_IIId, \
&&lb_count_i_bs_init_heap_bin_IId, \
&&lb_count_i_bs_init_heap_bin_heap_IIId, \
&&lb_count_i_bs_match_string_rfII, \
&&lb_count_i_bs_match_string_xfII, \
&&lb_count_i_bs_private_append_jId, \
&&lb_count_i_bs_put_utf16_jIs, \
&&lb_count_i_bs_put_utf8_js, \
&&lb_count_i_bs_restore2_rI, \
&&lb_count_i_bs_restore2_xI, \
&&lb_count_i_bs_save2_rI, \
&&lb_count_i_bs_save2_xI, \
&&lb_count_i_bs_skip_bits2_frxI, \
&&lb_count_i_bs_skip_bits2_fryI, \
&&lb_count_i_bs_skip_bits2_fxrI, \
&&lb_count_i_bs_skip_bits2_fxxI, \
&&lb_count_i_bs_skip_bits2_fxyI, \
&&lb_count_i_bs_skip_bits_all2_frI, \
&&lb_count_i_bs_skip_bits_all2_fxI, \
&&lb_count_i_bs_skip_bits_imm2_frI, \
&&lb_count_i_bs_skip_bits_imm2_fxI, \
&&lb_count_i_bs_start_match2_rfIId, \
&&lb_count_i_bs_start_match2_xfIId, \
&&lb_count_i_bs_start_match2_yfIId, \
&&lb_count_i_bs_utf16_size_sd, \
&&lb_count_i_bs_utf8_size_sd, \
&&lb_count_i_bs_validate_unicode_js, \
&&lb_count_i_bs_validate_unicode_retract_j, \
&&lb_count_i_bsl_jId, \
&&lb_count_i_bsr_jId, \
&&lb_count_i_bxor_jId, \
&&lb_count_i_call_f, \
&&lb_count_i_call_ext_e, \
&&lb_count_i_call_ext_last_eP, \
&&lb_count_i_call_ext_only_e, \
&&lb_count_i_call_fun_I, \
&&lb_count_i_call_fun_last_IP, \
&&lb_count_i_call_last_fP, \
&&lb_count_i_call_only_f, \
&&lb_count_i_debug_breakpoint, \
&&lb_count_i_element_rjsd, \
&&lb_count_i_element_xjsd, \
&&lb_count_i_element_yjsd, \
&&lb_count_i_fadd_lll, \
&&lb_count_i_fast_element_rjId, \
&&lb_count_i_fast_element_xjId, \
&&lb_count_i_fast_element_yjId, \
&&lb_count_i_fcheckerror, \
&&lb_count_i_fdiv_lll, \
&&lb_count_i_fetch_rx, \
&&lb_count_i_fetch_ry, \
&&lb_count_i_fetch_xr, \
&&lb_count_i_fetch_xx, \
&&lb_count_i_fetch_xy, \
&&lb_count_i_fetch_yr, \
&&lb_count_i_fetch_yx, \
&&lb_count_i_fetch_yy, \
&&lb_count_i_fetch_rc, \
&&lb_count_i_fetch_xc, \
&&lb_count_i_fetch_yc, \
&&lb_count_i_fetch_cr, \
&&lb_count_i_fetch_cx, \
&&lb_count_i_fetch_cy, \
&&lb_count_i_fetch_cc, \
&&lb_count_i_fetch_ss, \
&&lb_count_i_fmul_lll, \
&&lb_count_i_fnegate_ll, \
&&lb_count_i_fsub_lll, \
&&lb_count_i_func_info_IaaI, \
&&lb_count_i_gc_bif1_jIsId, \
&&lb_count_i_gc_bif2_jIId, \
&&lb_count_i_gc_bif3_jIsId, \
&&lb_count_i_generic_breakpoint, \
&&lb_count_i_get_sd, \
&&lb_count_i_get_map_element_frxr, \
&&lb_count_i_get_map_element_frxx, \
&&lb_count_i_get_map_element_frxy, \
&&lb_count_i_get_map_element_fxxr, \
&&lb_count_i_get_map_element_fxxx, \
&&lb_count_i_get_map_element_fxxy, \
&&lb_count_i_get_map_element_fyxr, \
&&lb_count_i_get_map_element_fyxx, \
&&lb_count_i_get_map_element_fyxy, \
&&lb_count_i_get_map_element_hash_frcIr, \
&&lb_count_i_get_map_element_hash_frcIx, \
&&lb_count_i_get_map_element_hash_frcIy, \
&&lb_count_i_get_map_element_hash_fxcIr, \
&&lb_count_i_get_map_element_hash_fxcIx, \
&&lb_count_i_get_map_element_hash_fxcIy, \
&&lb_count_i_get_map_element_hash_fycIr, \
&&lb_count_i_get_map_element_hash_fycIx, \
&&lb_count_i_get_map_element_hash_fycIy, \
&&lb_count_i_get_map_elements_fsI, \
&&lb_count_i_get_tuple_element_rPr, \
&&lb_count_i_get_tuple_element_rPx, \
&&lb_count_i_get_tuple_element_rPy, \
&&lb_count_i_get_tuple_element_xPr, \
&&lb_count_i_get_tuple_element_xPx, \
&&lb_count_i_get_tuple_element_xPy, \
&&lb_count_i_get_tuple_element_yPr, \
&&lb_count_i_get_tuple_element_yPx, \
&&lb_count_i_get_tuple_element_yPy, \
&&lb_count_i_hibernate, \
&&lb_count_i_increment_rIId, \
&&lb_count_i_increment_xIId, \
&&lb_count_i_increment_yIId, \
&&lb_count_i_int_bnot_jsId, \
&&lb_count_i_int_div_jId, \
&&lb_count_i_is_eq_f, \
&&lb_count_i_is_eq_exact_f, \
&&lb_count_i_is_eq_exact_immed_frc, \
&&lb_count_i_is_eq_exact_immed_fxc, \
&&lb_count_i_is_eq_exact_immed_fyc, \
&&lb_count_i_is_eq_exact_literal_rfc, \
&&lb_count_i_is_eq_exact_literal_xfc, \
&&lb_count_i_is_eq_exact_literal_yfc, \
&&lb_count_i_is_eq_exact_spec_frr, \
&&lb_count_i_is_eq_exact_spec_frx, \
&&lb_count_i_is_eq_exact_spec_fry, \
&&lb_count_i_is_eq_exact_spec_fxx, \
&&lb_count_i_is_eq_exact_spec_fxy, \
&&lb_count_i_is_ge_f, \
&&lb_count_i_is_ge_spec_fxx, \
&&lb_count_i_is_ge_spec_fxc, \
&&lb_count_i_is_ge_spec_fcx, \
&&lb_count_i_is_ge_spec_fcc, \
&&lb_count_i_is_lt_f, \
&&lb_count_i_is_lt_spec_frr, \
&&lb_count_i_is_lt_spec_frx, \
&&lb_count_i_is_lt_spec_fxr, \
&&lb_count_i_is_lt_spec_fxx, \
&&lb_count_i_is_lt_spec_frc, \
&&lb_count_i_is_lt_spec_fxc, \
&&lb_count_i_is_lt_spec_fcr, \
&&lb_count_i_is_lt_spec_fcx, \
&&lb_count_i_is_lt_spec_fcc, \
&&lb_count_i_is_ne_f, \
&&lb_count_i_is_ne_exact_f, \
&&lb_count_i_is_ne_exact_immed_frc, \
&&lb_count_i_is_ne_exact_immed_fxc, \
&&lb_count_i_is_ne_exact_immed_fyc, \
&&lb_count_i_is_ne_exact_literal_rfc, \
&&lb_count_i_is_ne_exact_literal_xfc, \
&&lb_count_i_is_ne_exact_literal_yfc, \
&&lb_count_i_jump_on_val_rfII, \
&&lb_count_i_jump_on_val_xfII, \
&&lb_count_i_jump_on_val_yfII, \
&&lb_count_i_jump_on_val_zero_rfI, \
&&lb_count_i_jump_on_val_zero_xfI, \
&&lb_count_i_jump_on_val_zero_yfI, \
&&lb_count_i_loop_rec_fr, \
&&lb_count_i_m_div_jId, \
&&lb_count_i_make_fun_It, \
&&lb_count_i_minus_jId, \
&&lb_count_i_minus_jIxxd, \
&&lb_count_i_move_call_crf, \
&&lb_count_i_move_call_ext_cre, \
&&lb_count_i_move_call_ext_last_ePcr, \
&&lb_count_i_move_call_ext_only_ecr, \
&&lb_count_i_move_call_last_fPcr, \
&&lb_count_i_move_call_only_fcr, \
&&lb_count_i_new_bs_put_binary_jsIs, \
&&lb_count_i_new_bs_put_binary_all_jsI, \
&&lb_count_i_new_bs_put_binary_imm_jIs, \
&&lb_count_i_new_bs_put_float_jsIs, \
&&lb_count_i_new_bs_put_float_imm_jIIs, \
&&lb_count_i_new_bs_put_integer_jsIs, \
&&lb_count_i_new_bs_put_integer_imm_jIIs, \
&&lb_count_i_plus_jId, \
&&lb_count_i_plus_jIxxd, \
&&lb_count_i_put_tuple_rI, \
&&lb_count_i_put_tuple_xI, \
&&lb_count_i_put_tuple_yI, \
&&lb_count_i_recv_set, \
&&lb_count_i_rem_jId, \
&&lb_count_i_rem_jIxxd, \
&&lb_count_i_return_time_trace, \
&&lb_count_i_return_to_trace, \
&&lb_count_i_select_tuple_arity_rfI, \
&&lb_count_i_select_tuple_arity_xfI, \
&&lb_count_i_select_tuple_arity_yfI, \
&&lb_count_i_select_tuple_arity2_rfAAff, \
&&lb_count_i_select_tuple_arity2_xfAAff, \
&&lb_count_i_select_tuple_arity2_yfAAff, \
&&lb_count_i_select_val2_rfccff, \
&&lb_count_i_select_val2_xfccff, \
&&lb_count_i_select_val2_yfccff, \
&&lb_count_i_select_val_bins_rfI, \
&&lb_count_i_select_val_bins_xfI, \
&&lb_count_i_select_val_bins_yfI, \
&&lb_count_i_select_val_lins_rfI, \
&&lb_count_i_select_val_lins_xfI, \
&&lb_count_i_select_val_lins_yfI, \
&&lb_count_i_times_jId, \
&&lb_count_i_trim_I, \
&&lb_count_i_wait_error, \
&&lb_count_i_wait_error_locked, \
&&lb_count_i_wait_timeout_fI, \
&&lb_count_i_wait_timeout_fs, \
&&lb_count_i_wait_timeout_locked_fI, \
&&lb_count_i_wait_timeout_locked_fs, \
&&lb_count_i_yield, \
&&lb_count_if_end, \
&&lb_count_init_y, \
&&lb_count_init2_yy, \
&&lb_count_init3_yyy, \
&&lb_count_int_code_end, \
&&lb_count_is_atom_fr, \
&&lb_count_is_atom_fx, \
&&lb_count_is_atom_fy, \
&&lb_count_is_binary_fr, \
&&lb_count_is_binary_fx, \
&&lb_count_is_binary_fy, \
&&lb_count_is_bitstring_fr, \
&&lb_count_is_bitstring_fx, \
&&lb_count_is_bitstring_fy, \
&&lb_count_is_boolean_fr, \
&&lb_count_is_boolean_fx, \
&&lb_count_is_boolean_fy, \
&&lb_count_is_float_fr, \
&&lb_count_is_float_fx, \
&&lb_count_is_float_fy, \
&&lb_count_is_function_fr, \
&&lb_count_is_function_fx, \
&&lb_count_is_function_fy, \
&&lb_count_is_function2_fss, \
&&lb_count_is_integer_fr, \
&&lb_count_is_integer_fx, \
&&lb_count_is_integer_fy, \
&&lb_count_is_integer_allocate_frII, \
&&lb_count_is_integer_allocate_fxII, \
&&lb_count_is_list_fr, \
&&lb_count_is_list_fx, \
&&lb_count_is_list_fy, \
&&lb_count_is_map_fr, \
&&lb_count_is_map_fx, \
&&lb_count_is_map_fy, \
&&lb_count_is_nil_fr, \
&&lb_count_is_nil_fx, \
&&lb_count_is_nil_fy, \
&&lb_count_is_non_empty_list_test_heap_frIt, \
&&lb_count_is_nonempty_list_fr, \
&&lb_count_is_nonempty_list_fx, \
&&lb_count_is_nonempty_list_fy, \
&&lb_count_is_nonempty_list_allocate_frIt, \
&&lb_count_is_nonempty_list_allocate_fxIt, \
&&lb_count_is_number_fr, \
&&lb_count_is_number_fx, \
&&lb_count_is_number_fy, \
&&lb_count_is_pid_fr, \
&&lb_count_is_pid_fx, \
&&lb_count_is_pid_fy, \
&&lb_count_is_port_fr, \
&&lb_count_is_port_fx, \
&&lb_count_is_port_fy, \
&&lb_count_is_reference_fr, \
&&lb_count_is_reference_fx, \
&&lb_count_is_reference_fy, \
&&lb_count_is_tuple_fr, \
&&lb_count_is_tuple_fx, \
&&lb_count_is_tuple_fy, \
&&lb_count_is_tuple_of_arity_frA, \
&&lb_count_is_tuple_of_arity_fxA, \
&&lb_count_is_tuple_of_arity_fyA, \
&&lb_count_jump_f, \
&&lb_count_label_L, \
&&lb_count_line_I, \
&&lb_count_loop_rec_end_f, \
&&lb_count_move_nr, \
&&lb_count_move_nx, \
&&lb_count_move_rx, \
&&lb_count_move_ry, \
&&lb_count_move_xr, \
&&lb_count_move_xx, \
&&lb_count_move_xy, \
&&lb_count_move_yr, \
&&lb_count_move_yx, \
&&lb_count_move_yy, \
&&lb_count_move_cr, \
&&lb_count_move_cx, \
&&lb_count_move2_rxyx, \
&&lb_count_move2_ryxy, \
&&lb_count_move2_xrxx, \
&&lb_count_move2_xrxy, \
&&lb_count_move2_xxxx, \
&&lb_count_move2_xyxy, \
&&lb_count_move2_yrxy, \
&&lb_count_move2_yxrx, \
&&lb_count_move2_yxyx, \
&&lb_count_move3_xxxxxx, \
&&lb_count_move3_xyxyxy, \
&&lb_count_move3_yxyxyx, \
&&lb_count_move_call_xrf, \
&&lb_count_move_call_yrf, \
&&lb_count_move_call_last_xrfQ, \
&&lb_count_move_call_last_yrfQ, \
&&lb_count_move_call_only_xrf, \
&&lb_count_move_deallocate_return_nrQ, \
&&lb_count_move_deallocate_return_xrQ, \
&&lb_count_move_deallocate_return_yrQ, \
&&lb_count_move_deallocate_return_crQ, \
&&lb_count_move_jump_fn, \
&&lb_count_move_jump_fx, \
&&lb_count_move_jump_fy, \
&&lb_count_move_jump_fc, \
&&lb_count_move_return_nr, \
&&lb_count_move_return_xr, \
&&lb_count_move_return_cr, \
&&lb_count_move_window3_xxxy, \
&&lb_count_move_window4_xxxxy, \
&&lb_count_move_window5_xxxxxy, \
&&lb_count_move_x1_c, \
&&lb_count_move_x2_c, \
&&lb_count_new_map_dII, \
&&lb_count_node_r, \
&&lb_count_node_x, \
&&lb_count_node_y, \
&&lb_count_normal_exit, \
&&lb_count_on_load, \
&&lb_count_put_list_rnx, \
&&lb_count_put_list_rxr, \
&&lb_count_put_list_rxx, \
&&lb_count_put_list_ryx, \
&&lb_count_put_list_xnx, \
&&lb_count_put_list_xrr, \
&&lb_count_put_list_xrx, \
&&lb_count_put_list_xxr, \
&&lb_count_put_list_xxx, \
&&lb_count_put_list_xyr, \
&&lb_count_put_list_xyx, \
&&lb_count_put_list_ynx, \
&&lb_count_put_list_yrr, \
&&lb_count_put_list_yrx, \
&&lb_count_put_list_yxr, \
&&lb_count_put_list_yxx, \
&&lb_count_put_list_yyr, \
&&lb_count_put_list_yyx, \
&&lb_count_put_list_rcr, \
&&lb_count_put_list_rcx, \
&&lb_count_put_list_rcy, \
&&lb_count_put_list_xcr, \
&&lb_count_put_list_xcx, \
&&lb_count_put_list_xcy, \
&&lb_count_put_list_ycr, \
&&lb_count_put_list_ycx, \
&&lb_count_put_list_ycy, \
&&lb_count_put_list_crr, \
&&lb_count_put_list_crx, \
&&lb_count_put_list_cry, \
&&lb_count_put_list_cxr, \
&&lb_count_put_list_cxx, \
&&lb_count_put_list_cxy, \
&&lb_count_put_list_cyr, \
&&lb_count_put_list_cyx, \
&&lb_count_put_list_cyy, \
&&lb_count_put_list_ssd, \
&&lb_count_raise_ss, \
&&lb_count_recv_mark_f, \
&&lb_count_remove_message, \
&&lb_count_return, \
&&lb_count_return_trace, \
&&lb_count_self_r, \
&&lb_count_self_x, \
&&lb_count_self_y, \
&&lb_count_send, \
&&lb_count_set_tuple_element_sdP, \
&&lb_count_system_limit_j, \
&&lb_count_test_arity_frA, \
&&lb_count_test_arity_fxA, \
&&lb_count_test_arity_fyA, \
&&lb_count_test_heap_It, \
&&lb_count_test_heap_1_put_list_Iy, \
&&lb_count_timeout, \
&&lb_count_timeout_locked, \
&&lb_count_try_case_end_s, \
&&lb_count_try_end_y, \
&&lb_count_update_map_assoc_jsdII, \
&&lb_count_update_map_exact_jsdII, \
&&lb_count_wait_f, \
&&lb_count_wait_locked_f, \
&&lb_count_wait_unlocked_f,

#define DEFINE_COUNTING_LABELS \
CountCase(allocate_tt): opc[0].count++; goto lb_allocate_tt; \
CountCase(allocate_heap_tIt): opc[1].count++; goto lb_allocate_heap_tIt; \
CountCase(allocate_heap_zero_tIt): opc[2].count++; goto lb_allocate_heap_zero_tIt; \
CountCase(allocate_init_tIy): opc[3].count++; goto lb_allocate_init_tIy; \
CountCase(allocate_zero_tt): opc[4].count++; goto lb_allocate_zero_tt; \
CountCase(apply_I): opc[5].count++; goto lb_apply_I; \
CountCase(apply_bif): opc[6].count++; goto lb_apply_bif; \
CountCase(apply_last_IP): opc[7].count++; goto lb_apply_last_IP; \
CountCase(badarg_j): opc[8].count++; goto lb_badarg_j; \
CountCase(badmatch_r): opc[9].count++; goto lb_badmatch_r; \
CountCase(badmatch_x): opc[10].count++; goto lb_badmatch_x; \
CountCase(badmatch_y): opc[11].count++; goto lb_badmatch_y; \
CountCase(bif1_fbsd): opc[12].count++; goto lb_bif1_fbsd; \
CountCase(bif1_body_bsd): opc[13].count++; goto lb_bif1_body_bsd; \
CountCase(bs_context_to_binary_r): opc[14].count++; goto lb_bs_context_to_binary_r; \
CountCase(bs_context_to_binary_x): opc[15].count++; goto lb_bs_context_to_binary_x; \
CountCase(bs_context_to_binary_y): opc[16].count++; goto lb_bs_context_to_binary_y; \
CountCase(bs_init_writable): opc[17].count++; goto lb_bs_init_writable; \
CountCase(bs_put_string_II): opc[18].count++; goto lb_bs_put_string_II; \
CountCase(bs_test_tail_imm2_frI): opc[19].count++; goto lb_bs_test_tail_imm2_frI; \
CountCase(bs_test_tail_imm2_fxI): opc[20].count++; goto lb_bs_test_tail_imm2_fxI; \
CountCase(bs_test_unit_frI): opc[21].count++; goto lb_bs_test_unit_frI; \
CountCase(bs_test_unit_fxI): opc[22].count++; goto lb_bs_test_unit_fxI; \
CountCase(bs_test_unit8_fr): opc[23].count++; goto lb_bs_test_unit8_fr; \
CountCase(bs_test_unit8_fx): opc[24].count++; goto lb_bs_test_unit8_fx; \
CountCase(bs_test_zero_tail2_fr): opc[25].count++; goto lb_bs_test_zero_tail2_fr; \
CountCase(bs_test_zero_tail2_fx): opc[26].count++; goto lb_bs_test_zero_tail2_fx; \
CountCase(call_bif_e): opc[27].count++; goto lb_call_bif_e; \
CountCase(call_error_handler): opc[28].count++; goto lb_call_error_handler; \
CountCase(call_nif): opc[29].count++; goto lb_call_nif; \
CountCase(case_end_r): opc[30].count++; goto lb_case_end_r; \
CountCase(case_end_x): opc[31].count++; goto lb_case_end_x; \
CountCase(case_end_y): opc[32].count++; goto lb_case_end_y; \
CountCase(catch_yf): opc[33].count++; goto lb_catch_yf; \
CountCase(catch_end_y): opc[34].count++; goto lb_catch_end_y; \
CountCase(continue_exit): opc[35].count++; goto lb_continue_exit; \
CountCase(deallocate_I): opc[36].count++; goto lb_deallocate_I; \
CountCase(deallocate_return_Q): opc[37].count++; goto lb_deallocate_return_Q; \
CountCase(error_action_code): opc[38].count++; goto lb_error_action_code; \
CountCase(extract_next_element_x): opc[39].count++; goto lb_extract_next_element_x; \
CountCase(extract_next_element_y): opc[40].count++; goto lb_extract_next_element_y; \
CountCase(extract_next_element2_x): opc[41].count++; goto lb_extract_next_element2_x; \
CountCase(extract_next_element2_y): opc[42].count++; goto lb_extract_next_element2_y; \
CountCase(extract_next_element3_x): opc[43].count++; goto lb_extract_next_element3_x; \
CountCase(extract_next_element3_y): opc[44].count++; goto lb_extract_next_element3_y; \
CountCase(fclearerror): opc[45].count++; goto lb_fclearerror; \
CountCase(fconv_dl): opc[46].count++; goto lb_fconv_dl; \
CountCase(fmove_ql): opc[47].count++; goto lb_fmove_ql; \
CountCase(fmove_ld): opc[48].count++; goto lb_fmove_ld; \
CountCase(fmove_dl): opc[49].count++; goto lb_fmove_dl; \
CountCase(get_list_rrx): opc[50].count++; goto lb_get_list_rrx; \
CountCase(get_list_rry): opc[51].count++; goto lb_get_list_rry; \
CountCase(get_list_rxr): opc[52].count++; goto lb_get_list_rxr; \
CountCase(get_list_rxx): opc[53].count++; goto lb_get_list_rxx; \
CountCase(get_list_rxy): opc[54].count++; goto lb_get_list_rxy; \
CountCase(get_list_ryr): opc[55].count++; goto lb_get_list_ryr; \
CountCase(get_list_ryx): opc[56].count++; goto lb_get_list_ryx; \
CountCase(get_list_ryy): opc[57].count++; goto lb_get_list_ryy; \
CountCase(get_list_xrx): opc[58].count++; goto lb_get_list_xrx; \
CountCase(get_list_xry): opc[59].count++; goto lb_get_list_xry; \
CountCase(get_list_xxr): opc[60].count++; goto lb_get_list_xxr; \
CountCase(get_list_xxx): opc[61].count++; goto lb_get_list_xxx; \
CountCase(get_list_xxy): opc[62].count++; goto lb_get_list_xxy; \
CountCase(get_list_xyr): opc[63].count++; goto lb_get_list_xyr; \
CountCase(get_list_xyx): opc[64].count++; goto lb_get_list_xyx; \
CountCase(get_list_xyy): opc[65].count++; goto lb_get_list_xyy; \
CountCase(get_list_yrx): opc[66].count++; goto lb_get_list_yrx; \
CountCase(get_list_yry): opc[67].count++; goto lb_get_list_yry; \
CountCase(get_list_yxr): opc[68].count++; goto lb_get_list_yxr; \
CountCase(get_list_yxx): opc[69].count++; goto lb_get_list_yxx; \
CountCase(get_list_yxy): opc[70].count++; goto lb_get_list_yxy; \
CountCase(get_list_yyr): opc[71].count++; goto lb_get_list_yyr; \
CountCase(get_list_yyx): opc[72].count++; goto lb_get_list_yyx; \
CountCase(get_list_yyy): opc[73].count++; goto lb_get_list_yyy; \
CountCase(hipe_call_count): opc[74].count++; goto lb_hipe_call_count; \
CountCase(hipe_trap_call): opc[75].count++; goto lb_hipe_trap_call; \
CountCase(hipe_trap_call_closure): opc[76].count++; goto lb_hipe_trap_call_closure; \
CountCase(hipe_trap_resume): opc[77].count++; goto lb_hipe_trap_resume; \
CountCase(hipe_trap_return): opc[78].count++; goto lb_hipe_trap_return; \
CountCase(hipe_trap_throw): opc[79].count++; goto lb_hipe_trap_throw; \
CountCase(i_apply): opc[80].count++; goto lb_i_apply; \
CountCase(i_apply_fun): opc[81].count++; goto lb_i_apply_fun; \
CountCase(i_apply_fun_last_P): opc[82].count++; goto lb_i_apply_fun_last_P; \
CountCase(i_apply_fun_only): opc[83].count++; goto lb_i_apply_fun_only; \
CountCase(i_apply_last_P): opc[84].count++; goto lb_i_apply_last_P; \
CountCase(i_apply_only): opc[85].count++; goto lb_i_apply_only; \
CountCase(i_band_jId): opc[86].count++; goto lb_i_band_jId; \
CountCase(i_band_jIxcd): opc[87].count++; goto lb_i_band_jIxcd; \
CountCase(i_bif2_fbd): opc[88].count++; goto lb_i_bif2_fbd; \
CountCase(i_bif2_body_bd): opc[89].count++; goto lb_i_bif2_body_bd; \
CountCase(i_bor_jId): opc[90].count++; goto lb_i_bor_jId; \
CountCase(i_bs_add_jId): opc[91].count++; goto lb_i_bs_add_jId; \
CountCase(i_bs_append_jIIId): opc[92].count++; goto lb_i_bs_append_jIIId; \
CountCase(i_bs_get_binary2_frIsId): opc[93].count++; goto lb_i_bs_get_binary2_frIsId; \
CountCase(i_bs_get_binary2_fxIsId): opc[94].count++; goto lb_i_bs_get_binary2_fxIsId; \
CountCase(i_bs_get_binary_all2_frIId): opc[95].count++; goto lb_i_bs_get_binary_all2_frIId; \
CountCase(i_bs_get_binary_all2_fxIId): opc[96].count++; goto lb_i_bs_get_binary_all2_fxIId; \
CountCase(i_bs_get_binary_all_reuse_rfI): opc[97].count++; goto lb_i_bs_get_binary_all_reuse_rfI; \
CountCase(i_bs_get_binary_all_reuse_xfI): opc[98].count++; goto lb_i_bs_get_binary_all_reuse_xfI; \
CountCase(i_bs_get_binary_imm2_frIIId): opc[99].count++; goto lb_i_bs_get_binary_imm2_frIIId; \
CountCase(i_bs_get_binary_imm2_fxIIId): opc[100].count++; goto lb_i_bs_get_binary_imm2_fxIIId; \
CountCase(i_bs_get_float2_frIsId): opc[101].count++; goto lb_i_bs_get_float2_frIsId; \
CountCase(i_bs_get_float2_fxIsId): opc[102].count++; goto lb_i_bs_get_float2_fxIsId; \
CountCase(i_bs_get_integer_fIId): opc[103].count++; goto lb_i_bs_get_integer_fIId; \
CountCase(i_bs_get_integer_16_rfd): opc[104].count++; goto lb_i_bs_get_integer_16_rfd; \
CountCase(i_bs_get_integer_16_xfd): opc[105].count++; goto lb_i_bs_get_integer_16_xfd; \
CountCase(i_bs_get_integer_32_rfId): opc[106].count++; goto lb_i_bs_get_integer_32_rfId; \
CountCase(i_bs_get_integer_32_xfId): opc[107].count++; goto lb_i_bs_get_integer_32_xfId; \
CountCase(i_bs_get_integer_8_rfd): opc[108].count++; goto lb_i_bs_get_integer_8_rfd; \
CountCase(i_bs_get_integer_8_xfd): opc[109].count++; goto lb_i_bs_get_integer_8_xfd; \
CountCase(i_bs_get_integer_imm_rIIfId): opc[110].count++; goto lb_i_bs_get_integer_imm_rIIfId; \
CountCase(i_bs_get_integer_imm_xIIfId): opc[111].count++; goto lb_i_bs_get_integer_imm_xIIfId; \
CountCase(i_bs_get_integer_small_imm_rIfId): opc[112].count++; goto lb_i_bs_get_integer_small_imm_rIfId; \
CountCase(i_bs_get_integer_small_imm_xIfId): opc[113].count++; goto lb_i_bs_get_integer_small_imm_xIfId; \
CountCase(i_bs_get_utf16_rfId): opc[114].count++; goto lb_i_bs_get_utf16_rfId; \
CountCase(i_bs_get_utf16_xfId): opc[115].count++; goto lb_i_bs_get_utf16_xfId; \
CountCase(i_bs_get_utf8_rfd): opc[116].count++; goto lb_i_bs_get_utf8_rfd; \
CountCase(i_bs_get_utf8_xfd): opc[117].count++; goto lb_i_bs_get_utf8_xfd; \
CountCase(i_bs_init_IId): opc[118].count++; goto lb_i_bs_init_IId; \
CountCase(i_bs_init_bits_IId): opc[119].count++; goto lb_i_bs_init_bits_IId; \
CountCase(i_bs_init_bits_fail_rjId): opc[120].count++; goto lb_i_bs_init_bits_fail_rjId; \
CountCase(i_bs_init_bits_fail_xjId): opc[121].count++; goto lb_i_bs_init_bits_fail_xjId; \
CountCase(i_bs_init_bits_fail_yjId): opc[122].count++; goto lb_i_bs_init_bits_fail_yjId; \
CountCase(i_bs_init_bits_fail_heap_IjId): opc[123].count++; goto lb_i_bs_init_bits_fail_heap_IjId; \
CountCase(i_bs_init_bits_heap_IIId): opc[124].count++; goto lb_i_bs_init_bits_heap_IIId; \
CountCase(i_bs_init_fail_rjId): opc[125].count++; goto lb_i_bs_init_fail_rjId; \
CountCase(i_bs_init_fail_xjId): opc[126].count++; goto lb_i_bs_init_fail_xjId; \
CountCase(i_bs_init_fail_yjId): opc[127].count++; goto lb_i_bs_init_fail_yjId; \
CountCase(i_bs_init_fail_heap_IjId): opc[128].count++; goto lb_i_bs_init_fail_heap_IjId; \
CountCase(i_bs_init_heap_IIId): opc[129].count++; goto lb_i_bs_init_heap_IIId; \
CountCase(i_bs_init_heap_bin_IId): opc[130].count++; goto lb_i_bs_init_heap_bin_IId; \
CountCase(i_bs_init_heap_bin_heap_IIId): opc[131].count++; goto lb_i_bs_init_heap_bin_heap_IIId; \
CountCase(i_bs_match_string_rfII): opc[132].count++; goto lb_i_bs_match_string_rfII; \
CountCase(i_bs_match_string_xfII): opc[133].count++; goto lb_i_bs_match_string_xfII; \
CountCase(i_bs_private_append_jId): opc[134].count++; goto lb_i_bs_private_append_jId; \
CountCase(i_bs_put_utf16_jIs): opc[135].count++; goto lb_i_bs_put_utf16_jIs; \
CountCase(i_bs_put_utf8_js): opc[136].count++; goto lb_i_bs_put_utf8_js; \
CountCase(i_bs_restore2_rI): opc[137].count++; goto lb_i_bs_restore2_rI; \
CountCase(i_bs_restore2_xI): opc[138].count++; goto lb_i_bs_restore2_xI; \
CountCase(i_bs_save2_rI): opc[139].count++; goto lb_i_bs_save2_rI; \
CountCase(i_bs_save2_xI): opc[140].count++; goto lb_i_bs_save2_xI; \
CountCase(i_bs_skip_bits2_frxI): opc[141].count++; goto lb_i_bs_skip_bits2_frxI; \
CountCase(i_bs_skip_bits2_fryI): opc[142].count++; goto lb_i_bs_skip_bits2_fryI; \
CountCase(i_bs_skip_bits2_fxrI): opc[143].count++; goto lb_i_bs_skip_bits2_fxrI; \
CountCase(i_bs_skip_bits2_fxxI): opc[144].count++; goto lb_i_bs_skip_bits2_fxxI; \
CountCase(i_bs_skip_bits2_fxyI): opc[145].count++; goto lb_i_bs_skip_bits2_fxyI; \
CountCase(i_bs_skip_bits_all2_frI): opc[146].count++; goto lb_i_bs_skip_bits_all2_frI; \
CountCase(i_bs_skip_bits_all2_fxI): opc[147].count++; goto lb_i_bs_skip_bits_all2_fxI; \
CountCase(i_bs_skip_bits_imm2_frI): opc[148].count++; goto lb_i_bs_skip_bits_imm2_frI; \
CountCase(i_bs_skip_bits_imm2_fxI): opc[149].count++; goto lb_i_bs_skip_bits_imm2_fxI; \
CountCase(i_bs_start_match2_rfIId): opc[150].count++; goto lb_i_bs_start_match2_rfIId; \
CountCase(i_bs_start_match2_xfIId): opc[151].count++; goto lb_i_bs_start_match2_xfIId; \
CountCase(i_bs_start_match2_yfIId): opc[152].count++; goto lb_i_bs_start_match2_yfIId; \
CountCase(i_bs_utf16_size_sd): opc[153].count++; goto lb_i_bs_utf16_size_sd; \
CountCase(i_bs_utf8_size_sd): opc[154].count++; goto lb_i_bs_utf8_size_sd; \
CountCase(i_bs_validate_unicode_js): opc[155].count++; goto lb_i_bs_validate_unicode_js; \
CountCase(i_bs_validate_unicode_retract_j): opc[156].count++; goto lb_i_bs_validate_unicode_retract_j; \
CountCase(i_bsl_jId): opc[157].count++; goto lb_i_bsl_jId; \
CountCase(i_bsr_jId): opc[158].count++; goto lb_i_bsr_jId; \
CountCase(i_bxor_jId): opc[159].count++; goto lb_i_bxor_jId; \
CountCase(i_call_f): opc[160].count++; goto lb_i_call_f; \
CountCase(i_call_ext_e): opc[161].count++; goto lb_i_call_ext_e; \
CountCase(i_call_ext_last_eP): opc[162].count++; goto lb_i_call_ext_last_eP; \
CountCase(i_call_ext_only_e): opc[163].count++; goto lb_i_call_ext_only_e; \
CountCase(i_call_fun_I): opc[164].count++; goto lb_i_call_fun_I; \
CountCase(i_call_fun_last_IP): opc[165].count++; goto lb_i_call_fun_last_IP; \
CountCase(i_call_last_fP): opc[166].count++; goto lb_i_call_last_fP; \
CountCase(i_call_only_f): opc[167].count++; goto lb_i_call_only_f; \
CountCase(i_debug_breakpoint): opc[168].count++; goto lb_i_debug_breakpoint; \
CountCase(i_element_rjsd): opc[169].count++; goto lb_i_element_rjsd; \
CountCase(i_element_xjsd): opc[170].count++; goto lb_i_element_xjsd; \
CountCase(i_element_yjsd): opc[171].count++; goto lb_i_element_yjsd; \
CountCase(i_fadd_lll): opc[172].count++; goto lb_i_fadd_lll; \
CountCase(i_fast_element_rjId): opc[173].count++; goto lb_i_fast_element_rjId; \
CountCase(i_fast_element_xjId): opc[174].count++; goto lb_i_fast_element_xjId; \
CountCase(i_fast_element_yjId): opc[175].count++; goto lb_i_fast_element_yjId; \
CountCase(i_fcheckerror): opc[176].count++; goto lb_i_fcheckerror; \
CountCase(i_fdiv_lll): opc[177].count++; goto lb_i_fdiv_lll; \
CountCase(i_fetch_rx): opc[178].count++; goto lb_i_fetch_rx; \
CountCase(i_fetch_ry): opc[179].count++; goto lb_i_fetch_ry; \
CountCase(i_fetch_xr): opc[180].count++; goto lb_i_fetch_xr; \
CountCase(i_fetch_xx): opc[181].count++; goto lb_i_fetch_xx; \
CountCase(i_fetch_xy): opc[182].count++; goto lb_i_fetch_xy; \
CountCase(i_fetch_yr): opc[183].count++; goto lb_i_fetch_yr; \
CountCase(i_fetch_yx): opc[184].count++; goto lb_i_fetch_yx; \
CountCase(i_fetch_yy): opc[185].count++; goto lb_i_fetch_yy; \
CountCase(i_fetch_rc): opc[186].count++; goto lb_i_fetch_rc; \
CountCase(i_fetch_xc): opc[187].count++; goto lb_i_fetch_xc; \
CountCase(i_fetch_yc): opc[188].count++; goto lb_i_fetch_yc; \
CountCase(i_fetch_cr): opc[189].count++; goto lb_i_fetch_cr; \
CountCase(i_fetch_cx): opc[190].count++; goto lb_i_fetch_cx; \
CountCase(i_fetch_cy): opc[191].count++; goto lb_i_fetch_cy; \
CountCase(i_fetch_cc): opc[192].count++; goto lb_i_fetch_cc; \
CountCase(i_fetch_ss): opc[193].count++; goto lb_i_fetch_ss; \
CountCase(i_fmul_lll): opc[194].count++; goto lb_i_fmul_lll; \
CountCase(i_fnegate_ll): opc[195].count++; goto lb_i_fnegate_ll; \
CountCase(i_fsub_lll): opc[196].count++; goto lb_i_fsub_lll; \
CountCase(i_func_info_IaaI): opc[197].count++; goto lb_i_func_info_IaaI; \
CountCase(i_gc_bif1_jIsId): opc[198].count++; goto lb_i_gc_bif1_jIsId; \
CountCase(i_gc_bif2_jIId): opc[199].count++; goto lb_i_gc_bif2_jIId; \
CountCase(i_gc_bif3_jIsId): opc[200].count++; goto lb_i_gc_bif3_jIsId; \
CountCase(i_generic_breakpoint): opc[201].count++; goto lb_i_generic_breakpoint; \
CountCase(i_get_sd): opc[202].count++; goto lb_i_get_sd; \
CountCase(i_get_map_element_frxr): opc[203].count++; goto lb_i_get_map_element_frxr; \
CountCase(i_get_map_element_frxx): opc[204].count++; goto lb_i_get_map_element_frxx; \
CountCase(i_get_map_element_frxy): opc[205].count++; goto lb_i_get_map_element_frxy; \
CountCase(i_get_map_element_fxxr): opc[206].count++; goto lb_i_get_map_element_fxxr; \
CountCase(i_get_map_element_fxxx): opc[207].count++; goto lb_i_get_map_element_fxxx; \
CountCase(i_get_map_element_fxxy): opc[208].count++; goto lb_i_get_map_element_fxxy; \
CountCase(i_get_map_element_fyxr): opc[209].count++; goto lb_i_get_map_element_fyxr; \
CountCase(i_get_map_element_fyxx): opc[210].count++; goto lb_i_get_map_element_fyxx; \
CountCase(i_get_map_element_fyxy): opc[211].count++; goto lb_i_get_map_element_fyxy; \
CountCase(i_get_map_element_hash_frcIr): opc[212].count++; goto lb_i_get_map_element_hash_frcIr; \
CountCase(i_get_map_element_hash_frcIx): opc[213].count++; goto lb_i_get_map_element_hash_frcIx; \
CountCase(i_get_map_element_hash_frcIy): opc[214].count++; goto lb_i_get_map_element_hash_frcIy; \
CountCase(i_get_map_element_hash_fxcIr): opc[215].count++; goto lb_i_get_map_element_hash_fxcIr; \
CountCase(i_get_map_element_hash_fxcIx): opc[216].count++; goto lb_i_get_map_element_hash_fxcIx; \
CountCase(i_get_map_element_hash_fxcIy): opc[217].count++; goto lb_i_get_map_element_hash_fxcIy; \
CountCase(i_get_map_element_hash_fycIr): opc[218].count++; goto lb_i_get_map_element_hash_fycIr; \
CountCase(i_get_map_element_hash_fycIx): opc[219].count++; goto lb_i_get_map_element_hash_fycIx; \
CountCase(i_get_map_element_hash_fycIy): opc[220].count++; goto lb_i_get_map_element_hash_fycIy; \
CountCase(i_get_map_elements_fsI): opc[221].count++; goto lb_i_get_map_elements_fsI; \
CountCase(i_get_tuple_element_rPr): opc[222].count++; goto lb_i_get_tuple_element_rPr; \
CountCase(i_get_tuple_element_rPx): opc[223].count++; goto lb_i_get_tuple_element_rPx; \
CountCase(i_get_tuple_element_rPy): opc[224].count++; goto lb_i_get_tuple_element_rPy; \
CountCase(i_get_tuple_element_xPr): opc[225].count++; goto lb_i_get_tuple_element_xPr; \
CountCase(i_get_tuple_element_xPx): opc[226].count++; goto lb_i_get_tuple_element_xPx; \
CountCase(i_get_tuple_element_xPy): opc[227].count++; goto lb_i_get_tuple_element_xPy; \
CountCase(i_get_tuple_element_yPr): opc[228].count++; goto lb_i_get_tuple_element_yPr; \
CountCase(i_get_tuple_element_yPx): opc[229].count++; goto lb_i_get_tuple_element_yPx; \
CountCase(i_get_tuple_element_yPy): opc[230].count++; goto lb_i_get_tuple_element_yPy; \
CountCase(i_hibernate): opc[231].count++; goto lb_i_hibernate; \
CountCase(i_increment_rIId): opc[232].count++; goto lb_i_increment_rIId; \
CountCase(i_increment_xIId): opc[233].count++; goto lb_i_increment_xIId; \
CountCase(i_increment_yIId): opc[234].count++; goto lb_i_increment_yIId; \
CountCase(i_int_bnot_jsId): opc[235].count++; goto lb_i_int_bnot_jsId; \
CountCase(i_int_div_jId): opc[236].count++; goto lb_i_int_div_jId; \
CountCase(i_is_eq_f): opc[237].count++; goto lb_i_is_eq_f; \
CountCase(i_is_eq_exact_f): opc[238].count++; goto lb_i_is_eq_exact_f; \
CountCase(i_is_eq_exact_immed_frc): opc[239].count++; goto lb_i_is_eq_exact_immed_frc; \
CountCase(i_is_eq_exact_immed_fxc): opc[240].count++; goto lb_i_is_eq_exact_immed_fxc; \
CountCase(i_is_eq_exact_immed_fyc): opc[241].count++; goto lb_i_is_eq_exact_immed_fyc; \
CountCase(i_is_eq_exact_literal_rfc): opc[242].count++; goto lb_i_is_eq_exact_literal_rfc; \
CountCase(i_is_eq_exact_literal_xfc): opc[243].count++; goto lb_i_is_eq_exact_literal_xfc; \
CountCase(i_is_eq_exact_literal_yfc): opc[244].count++; goto lb_i_is_eq_exact_literal_yfc; \
CountCase(i_is_eq_exact_spec_frr): opc[245].count++; goto lb_i_is_eq_exact_spec_frr; \
CountCase(i_is_eq_exact_spec_frx): opc[246].count++; goto lb_i_is_eq_exact_spec_frx; \
CountCase(i_is_eq_exact_spec_fry): opc[247].count++; goto lb_i_is_eq_exact_spec_fry; \
CountCase(i_is_eq_exact_spec_fxx): opc[248].count++; goto lb_i_is_eq_exact_spec_fxx; \
CountCase(i_is_eq_exact_spec_fxy): opc[249].count++; goto lb_i_is_eq_exact_spec_fxy; \
CountCase(i_is_ge_f): opc[250].count++; goto lb_i_is_ge_f; \
CountCase(i_is_ge_spec_fxx): opc[251].count++; goto lb_i_is_ge_spec_fxx; \
CountCase(i_is_ge_spec_fxc): opc[252].count++; goto lb_i_is_ge_spec_fxc; \
CountCase(i_is_ge_spec_fcx): opc[253].count++; goto lb_i_is_ge_spec_fcx; \
CountCase(i_is_ge_spec_fcc): opc[254].count++; goto lb_i_is_ge_spec_fcc; \
CountCase(i_is_lt_f): opc[255].count++; goto lb_i_is_lt_f; \
CountCase(i_is_lt_spec_frr): opc[256].count++; goto lb_i_is_lt_spec_frr; \
CountCase(i_is_lt_spec_frx): opc[257].count++; goto lb_i_is_lt_spec_frx; \
CountCase(i_is_lt_spec_fxr): opc[258].count++; goto lb_i_is_lt_spec_fxr; \
CountCase(i_is_lt_spec_fxx): opc[259].count++; goto lb_i_is_lt_spec_fxx; \
CountCase(i_is_lt_spec_frc): opc[260].count++; goto lb_i_is_lt_spec_frc; \
CountCase(i_is_lt_spec_fxc): opc[261].count++; goto lb_i_is_lt_spec_fxc; \
CountCase(i_is_lt_spec_fcr): opc[262].count++; goto lb_i_is_lt_spec_fcr; \
CountCase(i_is_lt_spec_fcx): opc[263].count++; goto lb_i_is_lt_spec_fcx; \
CountCase(i_is_lt_spec_fcc): opc[264].count++; goto lb_i_is_lt_spec_fcc; \
CountCase(i_is_ne_f): opc[265].count++; goto lb_i_is_ne_f; \
CountCase(i_is_ne_exact_f): opc[266].count++; goto lb_i_is_ne_exact_f; \
CountCase(i_is_ne_exact_immed_frc): opc[267].count++; goto lb_i_is_ne_exact_immed_frc; \
CountCase(i_is_ne_exact_immed_fxc): opc[268].count++; goto lb_i_is_ne_exact_immed_fxc; \
CountCase(i_is_ne_exact_immed_fyc): opc[269].count++; goto lb_i_is_ne_exact_immed_fyc; \
CountCase(i_is_ne_exact_literal_rfc): opc[270].count++; goto lb_i_is_ne_exact_literal_rfc; \
CountCase(i_is_ne_exact_literal_xfc): opc[271].count++; goto lb_i_is_ne_exact_literal_xfc; \
CountCase(i_is_ne_exact_literal_yfc): opc[272].count++; goto lb_i_is_ne_exact_literal_yfc; \
CountCase(i_jump_on_val_rfII): opc[273].count++; goto lb_i_jump_on_val_rfII; \
CountCase(i_jump_on_val_xfII): opc[274].count++; goto lb_i_jump_on_val_xfII; \
CountCase(i_jump_on_val_yfII): opc[275].count++; goto lb_i_jump_on_val_yfII; \
CountCase(i_jump_on_val_zero_rfI): opc[276].count++; goto lb_i_jump_on_val_zero_rfI; \
CountCase(i_jump_on_val_zero_xfI): opc[277].count++; goto lb_i_jump_on_val_zero_xfI; \
CountCase(i_jump_on_val_zero_yfI): opc[278].count++; goto lb_i_jump_on_val_zero_yfI; \
CountCase(i_loop_rec_fr): opc[279].count++; goto lb_i_loop_rec_fr; \
CountCase(i_m_div_jId): opc[280].count++; goto lb_i_m_div_jId; \
CountCase(i_make_fun_It): opc[281].count++; goto lb_i_make_fun_It; \
CountCase(i_minus_jId): opc[282].count++; goto lb_i_minus_jId; \
CountCase(i_minus_jIxxd): opc[283].count++; goto lb_i_minus_jIxxd; \
CountCase(i_move_call_crf): opc[284].count++; goto lb_i_move_call_crf; \
CountCase(i_move_call_ext_cre): opc[285].count++; goto lb_i_move_call_ext_cre; \
CountCase(i_move_call_ext_last_ePcr): opc[286].count++; goto lb_i_move_call_ext_last_ePcr; \
CountCase(i_move_call_ext_only_ecr): opc[287].count++; goto lb_i_move_call_ext_only_ecr; \
CountCase(i_move_call_last_fPcr): opc[288].count++; goto lb_i_move_call_last_fPcr; \
CountCase(i_move_call_only_fcr): opc[289].count++; goto lb_i_move_call_only_fcr; \
CountCase(i_new_bs_put_binary_jsIs): opc[290].count++; goto lb_i_new_bs_put_binary_jsIs; \
CountCase(i_new_bs_put_binary_all_jsI): opc[291].count++; goto lb_i_new_bs_put_binary_all_jsI; \
CountCase(i_new_bs_put_binary_imm_jIs): opc[292].count++; goto lb_i_new_bs_put_binary_imm_jIs; \
CountCase(i_new_bs_put_float_jsIs): opc[293].count++; goto lb_i_new_bs_put_float_jsIs; \
CountCase(i_new_bs_put_float_imm_jIIs): opc[294].count++; goto lb_i_new_bs_put_float_imm_jIIs; \
CountCase(i_new_bs_put_integer_jsIs): opc[295].count++; goto lb_i_new_bs_put_integer_jsIs; \
CountCase(i_new_bs_put_integer_imm_jIIs): opc[296].count++; goto lb_i_new_bs_put_integer_imm_jIIs; \
CountCase(i_plus_jId): opc[297].count++; goto lb_i_plus_jId; \
CountCase(i_plus_jIxxd): opc[298].count++; goto lb_i_plus_jIxxd; \
CountCase(i_put_tuple_rI): opc[299].count++; goto lb_i_put_tuple_rI; \
CountCase(i_put_tuple_xI): opc[300].count++; goto lb_i_put_tuple_xI; \
CountCase(i_put_tuple_yI): opc[301].count++; goto lb_i_put_tuple_yI; \
CountCase(i_recv_set): opc[302].count++; goto lb_i_recv_set; \
CountCase(i_rem_jId): opc[303].count++; goto lb_i_rem_jId; \
CountCase(i_rem_jIxxd): opc[304].count++; goto lb_i_rem_jIxxd; \
CountCase(i_return_time_trace): opc[305].count++; goto lb_i_return_time_trace; \
CountCase(i_return_to_trace): opc[306].count++; goto lb_i_return_to_trace; \
CountCase(i_select_tuple_arity_rfI): opc[307].count++; goto lb_i_select_tuple_arity_rfI; \
CountCase(i_select_tuple_arity_xfI): opc[308].count++; goto lb_i_select_tuple_arity_xfI; \
CountCase(i_select_tuple_arity_yfI): opc[309].count++; goto lb_i_select_tuple_arity_yfI; \
CountCase(i_select_tuple_arity2_rfAAff): opc[310].count++; goto lb_i_select_tuple_arity2_rfAAff; \
CountCase(i_select_tuple_arity2_xfAAff): opc[311].count++; goto lb_i_select_tuple_arity2_xfAAff; \
CountCase(i_select_tuple_arity2_yfAAff): opc[312].count++; goto lb_i_select_tuple_arity2_yfAAff; \
CountCase(i_select_val2_rfccff): opc[313].count++; goto lb_i_select_val2_rfccff; \
CountCase(i_select_val2_xfccff): opc[314].count++; goto lb_i_select_val2_xfccff; \
CountCase(i_select_val2_yfccff): opc[315].count++; goto lb_i_select_val2_yfccff; \
CountCase(i_select_val_bins_rfI): opc[316].count++; goto lb_i_select_val_bins_rfI; \
CountCase(i_select_val_bins_xfI): opc[317].count++; goto lb_i_select_val_bins_xfI; \
CountCase(i_select_val_bins_yfI): opc[318].count++; goto lb_i_select_val_bins_yfI; \
CountCase(i_select_val_lins_rfI): opc[319].count++; goto lb_i_select_val_lins_rfI; \
CountCase(i_select_val_lins_xfI): opc[320].count++; goto lb_i_select_val_lins_xfI; \
CountCase(i_select_val_lins_yfI): opc[321].count++; goto lb_i_select_val_lins_yfI; \
CountCase(i_times_jId): opc[322].count++; goto lb_i_times_jId; \
CountCase(i_trim_I): opc[323].count++; goto lb_i_trim_I; \
CountCase(i_wait_error): opc[324].count++; goto lb_i_wait_error; \
CountCase(i_wait_error_locked): opc[325].count++; goto lb_i_wait_error_locked; \
CountCase(i_wait_timeout_fI): opc[326].count++; goto lb_i_wait_timeout_fI; \
CountCase(i_wait_timeout_fs): opc[327].count++; goto lb_i_wait_timeout_fs; \
CountCase(i_wait_timeout_locked_fI): opc[328].count++; goto lb_i_wait_timeout_locked_fI; \
CountCase(i_wait_timeout_locked_fs): opc[329].count++; goto lb_i_wait_timeout_locked_fs; \
CountCase(i_yield): opc[330].count++; goto lb_i_yield; \
CountCase(if_end): opc[331].count++; goto lb_if_end; \
CountCase(init_y): opc[332].count++; goto lb_init_y; \
CountCase(init2_yy): opc[333].count++; goto lb_init2_yy; \
CountCase(init3_yyy): opc[334].count++; goto lb_init3_yyy; \
CountCase(int_code_end): opc[335].count++; goto lb_int_code_end; \
CountCase(is_atom_fr): opc[336].count++; goto lb_is_atom_fr; \
CountCase(is_atom_fx): opc[337].count++; goto lb_is_atom_fx; \
CountCase(is_atom_fy): opc[338].count++; goto lb_is_atom_fy; \
CountCase(is_binary_fr): opc[339].count++; goto lb_is_binary_fr; \
CountCase(is_binary_fx): opc[340].count++; goto lb_is_binary_fx; \
CountCase(is_binary_fy): opc[341].count++; goto lb_is_binary_fy; \
CountCase(is_bitstring_fr): opc[342].count++; goto lb_is_bitstring_fr; \
CountCase(is_bitstring_fx): opc[343].count++; goto lb_is_bitstring_fx; \
CountCase(is_bitstring_fy): opc[344].count++; goto lb_is_bitstring_fy; \
CountCase(is_boolean_fr): opc[345].count++; goto lb_is_boolean_fr; \
CountCase(is_boolean_fx): opc[346].count++; goto lb_is_boolean_fx; \
CountCase(is_boolean_fy): opc[347].count++; goto lb_is_boolean_fy; \
CountCase(is_float_fr): opc[348].count++; goto lb_is_float_fr; \
CountCase(is_float_fx): opc[349].count++; goto lb_is_float_fx; \
CountCase(is_float_fy): opc[350].count++; goto lb_is_float_fy; \
CountCase(is_function_fr): opc[351].count++; goto lb_is_function_fr; \
CountCase(is_function_fx): opc[352].count++; goto lb_is_function_fx; \
CountCase(is_function_fy): opc[353].count++; goto lb_is_function_fy; \
CountCase(is_function2_fss): opc[354].count++; goto lb_is_function2_fss; \
CountCase(is_integer_fr): opc[355].count++; goto lb_is_integer_fr; \
CountCase(is_integer_fx): opc[356].count++; goto lb_is_integer_fx; \
CountCase(is_integer_fy): opc[357].count++; goto lb_is_integer_fy; \
CountCase(is_integer_allocate_frII): opc[358].count++; goto lb_is_integer_allocate_frII; \
CountCase(is_integer_allocate_fxII): opc[359].count++; goto lb_is_integer_allocate_fxII; \
CountCase(is_list_fr): opc[360].count++; goto lb_is_list_fr; \
CountCase(is_list_fx): opc[361].count++; goto lb_is_list_fx; \
CountCase(is_list_fy): opc[362].count++; goto lb_is_list_fy; \
CountCase(is_map_fr): opc[363].count++; goto lb_is_map_fr; \
CountCase(is_map_fx): opc[364].count++; goto lb_is_map_fx; \
CountCase(is_map_fy): opc[365].count++; goto lb_is_map_fy; \
CountCase(is_nil_fr): opc[366].count++; goto lb_is_nil_fr; \
CountCase(is_nil_fx): opc[367].count++; goto lb_is_nil_fx; \
CountCase(is_nil_fy): opc[368].count++; goto lb_is_nil_fy; \
CountCase(is_non_empty_list_test_heap_frIt): opc[369].count++; goto lb_is_non_empty_list_test_heap_frIt; \
CountCase(is_nonempty_list_fr): opc[370].count++; goto lb_is_nonempty_list_fr; \
CountCase(is_nonempty_list_fx): opc[371].count++; goto lb_is_nonempty_list_fx; \
CountCase(is_nonempty_list_fy): opc[372].count++; goto lb_is_nonempty_list_fy; \
CountCase(is_nonempty_list_allocate_frIt): opc[373].count++; goto lb_is_nonempty_list_allocate_frIt; \
CountCase(is_nonempty_list_allocate_fxIt): opc[374].count++; goto lb_is_nonempty_list_allocate_fxIt; \
CountCase(is_number_fr): opc[375].count++; goto lb_is_number_fr; \
CountCase(is_number_fx): opc[376].count++; goto lb_is_number_fx; \
CountCase(is_number_fy): opc[377].count++; goto lb_is_number_fy; \
CountCase(is_pid_fr): opc[378].count++; goto lb_is_pid_fr; \
CountCase(is_pid_fx): opc[379].count++; goto lb_is_pid_fx; \
CountCase(is_pid_fy): opc[380].count++; goto lb_is_pid_fy; \
CountCase(is_port_fr): opc[381].count++; goto lb_is_port_fr; \
CountCase(is_port_fx): opc[382].count++; goto lb_is_port_fx; \
CountCase(is_port_fy): opc[383].count++; goto lb_is_port_fy; \
CountCase(is_reference_fr): opc[384].count++; goto lb_is_reference_fr; \
CountCase(is_reference_fx): opc[385].count++; goto lb_is_reference_fx; \
CountCase(is_reference_fy): opc[386].count++; goto lb_is_reference_fy; \
CountCase(is_tuple_fr): opc[387].count++; goto lb_is_tuple_fr; \
CountCase(is_tuple_fx): opc[388].count++; goto lb_is_tuple_fx; \
CountCase(is_tuple_fy): opc[389].count++; goto lb_is_tuple_fy; \
CountCase(is_tuple_of_arity_frA): opc[390].count++; goto lb_is_tuple_of_arity_frA; \
CountCase(is_tuple_of_arity_fxA): opc[391].count++; goto lb_is_tuple_of_arity_fxA; \
CountCase(is_tuple_of_arity_fyA): opc[392].count++; goto lb_is_tuple_of_arity_fyA; \
CountCase(jump_f): opc[393].count++; goto lb_jump_f; \
CountCase(label_L): opc[394].count++; goto lb_label_L; \
CountCase(line_I): opc[395].count++; goto lb_line_I; \
CountCase(loop_rec_end_f): opc[396].count++; goto lb_loop_rec_end_f; \
CountCase(move_nr): opc[397].count++; goto lb_move_nr; \
CountCase(move_nx): opc[398].count++; goto lb_move_nx; \
CountCase(move_rx): opc[399].count++; goto lb_move_rx; \
CountCase(move_ry): opc[400].count++; goto lb_move_ry; \
CountCase(move_xr): opc[401].count++; goto lb_move_xr; \
CountCase(move_xx): opc[402].count++; goto lb_move_xx; \
CountCase(move_xy): opc[403].count++; goto lb_move_xy; \
CountCase(move_yr): opc[404].count++; goto lb_move_yr; \
CountCase(move_yx): opc[405].count++; goto lb_move_yx; \
CountCase(move_yy): opc[406].count++; goto lb_move_yy; \
CountCase(move_cr): opc[407].count++; goto lb_move_cr; \
CountCase(move_cx): opc[408].count++; goto lb_move_cx; \
CountCase(move2_rxyx): opc[409].count++; goto lb_move2_rxyx; \
CountCase(move2_ryxy): opc[410].count++; goto lb_move2_ryxy; \
CountCase(move2_xrxx): opc[411].count++; goto lb_move2_xrxx; \
CountCase(move2_xrxy): opc[412].count++; goto lb_move2_xrxy; \
CountCase(move2_xxxx): opc[413].count++; goto lb_move2_xxxx; \
CountCase(move2_xyxy): opc[414].count++; goto lb_move2_xyxy; \
CountCase(move2_yrxy): opc[415].count++; goto lb_move2_yrxy; \
CountCase(move2_yxrx): opc[416].count++; goto lb_move2_yxrx; \
CountCase(move2_yxyx): opc[417].count++; goto lb_move2_yxyx; \
CountCase(move3_xxxxxx): opc[418].count++; goto lb_move3_xxxxxx; \
CountCase(move3_xyxyxy): opc[419].count++; goto lb_move3_xyxyxy; \
CountCase(move3_yxyxyx): opc[420].count++; goto lb_move3_yxyxyx; \
CountCase(move_call_xrf): opc[421].count++; goto lb_move_call_xrf; \
CountCase(move_call_yrf): opc[422].count++; goto lb_move_call_yrf; \
CountCase(move_call_last_xrfQ): opc[423].count++; goto lb_move_call_last_xrfQ; \
CountCase(move_call_last_yrfQ): opc[424].count++; goto lb_move_call_last_yrfQ; \
CountCase(move_call_only_xrf): opc[425].count++; goto lb_move_call_only_xrf; \
CountCase(move_deallocate_return_nrQ): opc[426].count++; goto lb_move_deallocate_return_nrQ; \
CountCase(move_deallocate_return_xrQ): opc[427].count++; goto lb_move_deallocate_return_xrQ; \
CountCase(move_deallocate_return_yrQ): opc[428].count++; goto lb_move_deallocate_return_yrQ; \
CountCase(move_deallocate_return_crQ): opc[429].count++; goto lb_move_deallocate_return_crQ; \
CountCase(move_jump_fn): opc[430].count++; goto lb_move_jump_fn; \
CountCase(move_jump_fx): opc[431].count++; goto lb_move_jump_fx; \
CountCase(move_jump_fy): opc[432].count++; goto lb_move_jump_fy; \
CountCase(move_jump_fc): opc[433].count++; goto lb_move_jump_fc; \
CountCase(move_return_nr): opc[434].count++; goto lb_move_return_nr; \
CountCase(move_return_xr): opc[435].count++; goto lb_move_return_xr; \
CountCase(move_return_cr): opc[436].count++; goto lb_move_return_cr; \
CountCase(move_window3_xxxy): opc[437].count++; goto lb_move_window3_xxxy; \
CountCase(move_window4_xxxxy): opc[438].count++; goto lb_move_window4_xxxxy; \
CountCase(move_window5_xxxxxy): opc[439].count++; goto lb_move_window5_xxxxxy; \
CountCase(move_x1_c): opc[440].count++; goto lb_move_x1_c; \
CountCase(move_x2_c): opc[441].count++; goto lb_move_x2_c; \
CountCase(new_map_dII): opc[442].count++; goto lb_new_map_dII; \
CountCase(node_r): opc[443].count++; goto lb_node_r; \
CountCase(node_x): opc[444].count++; goto lb_node_x; \
CountCase(node_y): opc[445].count++; goto lb_node_y; \
CountCase(normal_exit): opc[446].count++; goto lb_normal_exit; \
CountCase(on_load): opc[447].count++; goto lb_on_load; \
CountCase(put_list_rnx): opc[448].count++; goto lb_put_list_rnx; \
CountCase(put_list_rxr): opc[449].count++; goto lb_put_list_rxr; \
CountCase(put_list_rxx): opc[450].count++; goto lb_put_list_rxx; \
CountCase(put_list_ryx): opc[451].count++; goto lb_put_list_ryx; \
CountCase(put_list_xnx): opc[452].count++; goto lb_put_list_xnx; \
CountCase(put_list_xrr): opc[453].count++; goto lb_put_list_xrr; \
CountCase(put_list_xrx): opc[454].count++; goto lb_put_list_xrx; \
CountCase(put_list_xxr): opc[455].count++; goto lb_put_list_xxr; \
CountCase(put_list_xxx): opc[456].count++; goto lb_put_list_xxx; \
CountCase(put_list_xyr): opc[457].count++; goto lb_put_list_xyr; \
CountCase(put_list_xyx): opc[458].count++; goto lb_put_list_xyx; \
CountCase(put_list_ynx): opc[459].count++; goto lb_put_list_ynx; \
CountCase(put_list_yrr): opc[460].count++; goto lb_put_list_yrr; \
CountCase(put_list_yrx): opc[461].count++; goto lb_put_list_yrx; \
CountCase(put_list_yxr): opc[462].count++; goto lb_put_list_yxr; \
CountCase(put_list_yxx): opc[463].count++; goto lb_put_list_yxx; \
CountCase(put_list_yyr): opc[464].count++; goto lb_put_list_yyr; \
CountCase(put_list_yyx): opc[465].count++; goto lb_put_list_yyx; \
CountCase(put_list_rcr): opc[466].count++; goto lb_put_list_rcr; \
CountCase(put_list_rcx): opc[467].count++; goto lb_put_list_rcx; \
CountCase(put_list_rcy): opc[468].count++; goto lb_put_list_rcy; \
CountCase(put_list_xcr): opc[469].count++; goto lb_put_list_xcr; \
CountCase(put_list_xcx): opc[470].count++; goto lb_put_list_xcx; \
CountCase(put_list_xcy): opc[471].count++; goto lb_put_list_xcy; \
CountCase(put_list_ycr): opc[472].count++; goto lb_put_list_ycr; \
CountCase(put_list_ycx): opc[473].count++; goto lb_put_list_ycx; \
CountCase(put_list_ycy): opc[474].count++; goto lb_put_list_ycy; \
CountCase(put_list_crr): opc[475].count++; goto lb_put_list_crr; \
CountCase(put_list_crx): opc[476].count++; goto lb_put_list_crx; \
CountCase(put_list_cry): opc[477].count++; goto lb_put_list_cry; \
CountCase(put_list_cxr): opc[478].count++; goto lb_put_list_cxr; \
CountCase(put_list_cxx): opc[479].count++; goto lb_put_list_cxx; \
CountCase(put_list_cxy): opc[480].count++; goto lb_put_list_cxy; \
CountCase(put_list_cyr): opc[481].count++; goto lb_put_list_cyr; \
CountCase(put_list_cyx): opc[482].count++; goto lb_put_list_cyx; \
CountCase(put_list_cyy): opc[483].count++; goto lb_put_list_cyy; \
CountCase(put_list_ssd): opc[484].count++; goto lb_put_list_ssd; \
CountCase(raise_ss): opc[485].count++; goto lb_raise_ss; \
CountCase(recv_mark_f): opc[486].count++; goto lb_recv_mark_f; \
CountCase(remove_message): opc[487].count++; goto lb_remove_message; \
CountCase(return): opc[488].count++; goto lb_return; \
CountCase(return_trace): opc[489].count++; goto lb_return_trace; \
CountCase(self_r): opc[490].count++; goto lb_self_r; \
CountCase(self_x): opc[491].count++; goto lb_self_x; \
CountCase(self_y): opc[492].count++; goto lb_self_y; \
CountCase(send): opc[493].count++; goto lb_send; \
CountCase(set_tuple_element_sdP): opc[494].count++; goto lb_set_tuple_element_sdP; \
CountCase(system_limit_j): opc[495].count++; goto lb_system_limit_j; \
CountCase(test_arity_frA): opc[496].count++; goto lb_test_arity_frA; \
CountCase(test_arity_fxA): opc[497].count++; goto lb_test_arity_fxA; \
CountCase(test_arity_fyA): opc[498].count++; goto lb_test_arity_fyA; \
CountCase(test_heap_It): opc[499].count++; goto lb_test_heap_It; \
CountCase(test_heap_1_put_list_Iy): opc[500].count++; goto lb_test_heap_1_put_list_Iy; \
CountCase(timeout): opc[501].count++; goto lb_timeout; \
CountCase(timeout_locked): opc[502].count++; goto lb_timeout_locked; \
CountCase(try_case_end_s): opc[503].count++; goto lb_try_case_end_s; \
CountCase(try_end_y): opc[504].count++; goto lb_try_end_y; \
CountCase(update_map_assoc_jsdII): opc[505].count++; goto lb_update_map_assoc_jsdII; \
CountCase(update_map_exact_jsdII): opc[506].count++; goto lb_update_map_exact_jsdII; \
CountCase(wait_f): opc[507].count++; goto lb_wait_f; \
CountCase(wait_locked_f): opc[508].count++; goto lb_wait_locked_f; \
CountCase(wait_unlocked_f): opc[509].count++; goto lb_wait_unlocked_f;

#define genop_label_1 1
#define genop_func_info_3 2
#define genop_int_code_end_0 3
#define genop_call_2 4
#define genop_call_last_3 5
#define genop_call_only_2 6
#define genop_call_ext_2 7
#define genop_call_ext_last_3 8
#define genop_bif0_2 9
#define genop_bif1_4 10
#define genop_bif2_5 11
#define genop_allocate_2 12
#define genop_allocate_heap_3 13
#define genop_allocate_zero_2 14
#define genop_allocate_heap_zero_3 15
#define genop_test_heap_2 16
#define genop_init_1 17
#define genop_deallocate_1 18
#define genop_return_0 19
#define genop_send_0 20
#define genop_remove_message_0 21
#define genop_timeout_0 22
#define genop_loop_rec_2 23
#define genop_loop_rec_end_1 24
#define genop_wait_1 25
#define genop_wait_timeout_2 26
#define genop_m_plus_4 27
#define genop_m_minus_4 28
#define genop_m_times_4 29
#define genop_m_div_4 30
#define genop_int_div_4 31
#define genop_int_rem_4 32
#define genop_int_band_4 33
#define genop_int_bor_4 34
#define genop_int_bxor_4 35
#define genop_int_bsl_4 36
#define genop_int_bsr_4 37
#define genop_int_bnot_3 38
#define genop_is_lt_3 39
#define genop_is_ge_3 40
#define genop_is_eq_3 41
#define genop_is_ne_3 42
#define genop_is_eq_exact_3 43
#define genop_is_ne_exact_3 44
#define genop_is_integer_2 45
#define genop_is_float_2 46
#define genop_is_number_2 47
#define genop_is_atom_2 48
#define genop_is_pid_2 49
#define genop_is_reference_2 50
#define genop_is_port_2 51
#define genop_is_nil_2 52
#define genop_is_binary_2 53
#define genop_is_constant_2 54
#define genop_is_list_2 55
#define genop_is_nonempty_list_2 56
#define genop_is_tuple_2 57
#define genop_test_arity_3 58
#define genop_select_val_3 59
#define genop_select_tuple_arity_3 60
#define genop_jump_1 61
#define genop_catch_2 62
#define genop_catch_end_1 63
#define genop_move_2 64
#define genop_get_list_3 65
#define genop_get_tuple_element_3 66
#define genop_set_tuple_element_3 67
#define genop_put_string_3 68
#define genop_put_list_3 69
#define genop_put_tuple_2 70
#define genop_put_1 71
#define genop_badmatch_1 72
#define genop_if_end_0 73
#define genop_case_end_1 74
#define genop_call_fun_1 75
#define genop_make_fun_3 76
#define genop_is_function_2 77
#define genop_call_ext_only_2 78
#define genop_bs_start_match_2 79
#define genop_bs_get_integer_5 80
#define genop_bs_get_float_5 81
#define genop_bs_get_binary_5 82
#define genop_bs_skip_bits_4 83
#define genop_bs_test_tail_2 84
#define genop_bs_save_1 85
#define genop_bs_restore_1 86
#define genop_bs_init_2 87
#define genop_bs_final_2 88
#define genop_bs_put_integer_5 89
#define genop_bs_put_binary_5 90
#define genop_bs_put_float_5 91
#define genop_bs_put_string_2 92
#define genop_bs_need_buf_1 93
#define genop_fclearerror_0 94
#define genop_fcheckerror_1 95
#define genop_fmove_2 96
#define genop_fconv_2 97
#define genop_fadd_4 98
#define genop_fsub_4 99
#define genop_fmul_4 100
#define genop_fdiv_4 101
#define genop_fnegate_3 102
#define genop_make_fun2_1 103
#define genop_try_2 104
#define genop_try_end_1 105
#define genop_try_case_1 106
#define genop_try_case_end_1 107
#define genop_raise_2 108
#define genop_bs_init2_6 109
#define genop_bs_bits_to_bytes_3 110
#define genop_bs_add_5 111
#define genop_apply_1 112
#define genop_apply_last_2 113
#define genop_is_boolean_2 114
#define genop_is_function2_3 115
#define genop_bs_start_match2_5 116
#define genop_bs_get_integer2_7 117
#define genop_bs_get_float2_7 118
#define genop_bs_get_binary2_7 119
#define genop_bs_skip_bits2_5 120
#define genop_bs_test_tail2_3 121
#define genop_bs_save2_2 122
#define genop_bs_restore2_2 123
#define genop_gc_bif1_5 124
#define genop_gc_bif2_6 125
#define genop_bs_final2_2 126
#define genop_bs_bits_to_bytes2_2 127
#define genop_put_literal_2 128
#define genop_is_bitstr_2 129
#define genop_bs_context_to_binary_1 130
#define genop_bs_test_unit_3 131
#define genop_bs_match_string_4 132
#define genop_bs_init_writable_0 133
#define genop_bs_append_8 134
#define genop_bs_private_append_6 135
#define genop_trim_2 136
#define genop_bs_init_bits_6 137
#define genop_bs_get_utf8_5 138
#define genop_bs_skip_utf8_4 139
#define genop_bs_get_utf16_5 140
#define genop_bs_skip_utf16_4 141
#define genop_bs_get_utf32_5 142
#define genop_bs_skip_utf32_4 143
#define genop_bs_utf8_size_3 144
#define genop_bs_put_utf8_3 145
#define genop_bs_utf16_size_3 146
#define genop_bs_put_utf16_3 147
#define genop_bs_put_utf32_3 148
#define genop_on_load_0 149
#define genop_recv_mark_1 150
#define genop_recv_set_1 151
#define genop_gc_bif3_7 152
#define genop_line_1 153
#define genop_put_map_assoc_5 154
#define genop_put_map_exact_5 155
#define genop_is_map_2 156
#define genop_has_map_fields_3 157
#define genop_get_map_elements_3 158
#define genop_too_old_compiler_0 159
#define genop_i_func_info_4 160
#define genop_i_generic_breakpoint_0 161
#define genop_i_debug_breakpoint_0 162
#define genop_i_return_time_trace_0 163
#define genop_i_return_to_trace_0 164
#define genop_i_yield_0 165
#define genop_i_trim_1 166
#define genop_init2_2 167
#define genop_init3_3 168
#define genop_i_select_val_bins_3 169
#define genop_i_select_val_lins_3 170
#define genop_i_select_val2_6 171
#define genop_i_select_tuple_arity_3 172
#define genop_i_select_tuple_arity2_6 173
#define genop_i_jump_on_val_zero_3 174
#define genop_i_jump_on_val_4 175
#define genop_i_get_tuple_element_3 176
#define genop_badarg_1 177
#define genop_system_limit_1 178
#define genop_move_jump_2 179
#define genop_move_window_5 180
#define genop_move_window_6 181
#define genop_move_window3_4 182
#define genop_move_window4_5 183
#define genop_move_window5_6 184
#define genop_move_x1_1 185
#define genop_move_x2_1 186
#define genop_move2_4 187
#define genop_move3_6 188
#define genop_timeout_locked_0 189
#define genop_i_loop_rec_2 190
#define genop_wait_locked_1 191
#define genop_wait_unlocked_1 192
#define genop_i_wait_timeout_2 193
#define genop_i_wait_timeout_locked_2 194
#define genop_i_wait_error_0 195
#define genop_i_wait_error_locked_0 196
#define genop_i_is_eq_exact_immed_3 197
#define genop_i_is_eq_exact_literal_3 198
#define genop_i_is_ne_exact_immed_3 199
#define genop_i_is_ne_exact_literal_3 200
#define genop_i_is_eq_exact_spec_3 201
#define genop_i_is_lt_spec_3 202
#define genop_i_is_ge_spec_3 203
#define genop_i_is_eq_exact_1 204
#define genop_i_is_ne_exact_1 205
#define genop_i_is_lt_1 206
#define genop_i_is_ge_1 207
#define genop_i_is_eq_1 208
#define genop_i_is_ne_1 209
#define genop_i_put_tuple_2 210
#define genop_i_fetch_2 211
#define genop_normal_exit_0 212
#define genop_continue_exit_0 213
#define genop_apply_bif_0 214
#define genop_call_nif_0 215
#define genop_call_error_handler_0 216
#define genop_error_action_code_0 217
#define genop_return_trace_0 218
#define genop_move_return_2 219
#define genop_move_deallocate_return_3 220
#define genop_deallocate_return_1 221
#define genop_test_heap_1_put_list_2 222
#define genop_is_tuple_of_arity_3 223
#define genop_original_reg_2 224
#define genop_extract_next_element_1 225
#define genop_extract_next_element2_1 226
#define genop_extract_next_element3_1 227
#define genop_is_integer_allocate_4 228
#define genop_is_nonempty_list_allocate_4 229
#define genop_is_non_empty_list_test_heap_4 230
#define genop_is_bitstring_2 231
#define genop_allocate_init_3 232
#define genop_i_apply_0 233
#define genop_i_apply_last_1 234
#define genop_i_apply_only_0 235
#define genop_i_apply_fun_0 236
#define genop_i_apply_fun_last_1 237
#define genop_i_apply_fun_only_0 238
#define genop_i_hibernate_0 239
#define genop_call_bif_1 240
#define genop_i_get_2 241
#define genop_self_1 242
#define genop_node_1 243
#define genop_i_fast_element_4 244
#define genop_i_element_4 245
#define genop_bif1_body_3 246
#define genop_i_bif2_3 247
#define genop_i_bif2_body_2 248
#define genop_i_move_call_3 249
#define genop_move_call_3 250
#define genop_i_move_call_last_4 251
#define genop_move_call_last_4 252
#define genop_i_move_call_only_3 253
#define genop_move_call_only_3 254
#define genop_i_call_1 255
#define genop_i_call_last_2 256
#define genop_i_call_only_1 257
#define genop_i_call_ext_1 258
#define genop_i_call_ext_last_2 259
#define genop_i_call_ext_only_1 260
#define genop_i_move_call_ext_3 261
#define genop_i_move_call_ext_last_4 262
#define genop_i_move_call_ext_only_3 263
#define genop_i_call_fun_1 264
#define genop_i_call_fun_last_2 265
#define genop_i_make_fun_2 266
#define genop_i_bs_start_match2_5 267
#define genop_i_bs_save2_2 268
#define genop_i_bs_restore2_2 269
#define genop_i_bs_match_string_4 270
#define genop_i_bs_get_integer_small_imm_5 271
#define genop_i_bs_get_integer_imm_6 272
#define genop_i_bs_get_integer_4 273
#define genop_i_bs_get_integer_8_3 274
#define genop_i_bs_get_integer_16_3 275
#define genop_i_bs_get_integer_32_4 276
#define genop_i_bs_get_binary_imm2_6 277
#define genop_i_bs_get_binary2_6 278
#define genop_i_bs_get_binary_all2_5 279
#define genop_i_bs_get_binary_all_reuse_3 280
#define genop_i_bs_get_float2_6 281
#define genop_i_bs_skip_bits_imm2_3 282
#define genop_i_bs_skip_bits2_4 283
#define genop_i_bs_skip_bits_all2_3 284
#define genop_bs_test_zero_tail2_2 285
#define genop_bs_test_tail_imm2_3 286
#define genop_bs_test_unit8_2 287
#define genop_i_bs_get_utf8_3 288
#define genop_i_bs_get_utf16_4 289
#define genop_i_bs_validate_unicode_retract_1 290
#define genop_i_bs_init_fail_4 291
#define genop_i_bs_init_fail_heap_4 292
#define genop_i_bs_init_3 293
#define genop_i_bs_init_heap_bin_3 294
#define genop_i_bs_init_heap_4 295
#define genop_i_bs_init_heap_bin_heap_4 296
#define genop_i_bs_init_bits_fail_4 297
#define genop_i_bs_init_bits_fail_heap_4 298
#define genop_i_bs_init_bits_3 299
#define genop_i_bs_init_bits_heap_4 300
#define genop_i_bs_add_3 301
#define genop_i_bs_append_5 302
#define genop_i_bs_private_append_3 303
#define genop_i_new_bs_put_integer_4 304
#define genop_i_new_bs_put_integer_imm_4 305
#define genop_i_bs_utf8_size_2 306
#define genop_i_bs_utf16_size_2 307
#define genop_i_bs_put_utf8_2 308
#define genop_i_bs_put_utf16_3 309
#define genop_i_bs_validate_unicode_2 310
#define genop_i_new_bs_put_float_4 311
#define genop_i_new_bs_put_float_imm_4 312
#define genop_i_new_bs_put_binary_4 313
#define genop_i_new_bs_put_binary_imm_3 314
#define genop_i_new_bs_put_binary_all_3 315
#define genop_i_fadd_3 316
#define genop_i_fsub_3 317
#define genop_i_fmul_3 318
#define genop_i_fdiv_3 319
#define genop_i_fnegate_2 320
#define genop_i_fcheckerror_0 321
#define genop_sorted_put_map_assoc_5 322
#define genop_sorted_put_map_exact_5 323
#define genop_new_map_3 324
#define genop_update_map_assoc_5 325
#define genop_update_map_exact_5 326
#define genop_i_get_map_elements_3 327
#define genop_i_get_map_element_hash_5 328
#define genop_i_get_map_element_4 329
#define genop_i_increment_4 330
#define genop_i_plus_5 331
#define genop_i_plus_3 332
#define genop_i_minus_5 333
#define genop_i_minus_3 334
#define genop_i_times_3 335
#define genop_i_m_div_3 336
#define genop_i_int_div_3 337
#define genop_i_rem_5 338
#define genop_i_rem_3 339
#define genop_i_bsl_3 340
#define genop_i_bsr_3 341
#define genop_i_band_5 342
#define genop_i_band_3 343
#define genop_i_bor_3 344
#define genop_i_bxor_3 345
#define genop_i_int_bnot_4 346
#define genop_i_gc_bif1_5 347
#define genop_ii_gc_bif2_6 348
#define genop_i_gc_bif2_4 349
#define genop_ii_gc_bif3_7 350
#define genop_i_gc_bif3_5 351
#define genop_unsupported_guard_bif_3 352
#define genop_i_recv_set_0 353
#define genop_hipe_trap_call_0 354
#define genop_hipe_trap_call_closure_0 355
#define genop_hipe_trap_return_0 356
#define genop_hipe_trap_throw_0 357
#define genop_hipe_trap_resume_0 358
#define genop_hipe_call_count_0 359
#endif
# 3008 "armv6-portbld-freebsd11.0/opt/smp/beam_opcodes.h"
# 25 "beam/beam_load.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/beam_load.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 1 "beam/erl_process_lock.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2007-2012. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */


/*
 * Description:	Impementation of Erlang process locks.
 *
 * Author: 	Rickard Green
 */

#ifndef ERTS_PROC_LOCK_TYPE__
#define ERTS_PROC_LOCK_TYPE__

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_PROC_LOCK_DEBUG
#endif
# 34 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_count.h"
#endif /* expanded by -frewrite-includes */
# 36 "beam/erl_process_lock.h"
# 37 "beam/erl_process_lock.h"
#endif
# 38 "beam/erl_process_lock.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 39 "beam/erl_process_lock.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 29 "beam/erl_smp.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 40 "beam/erl_process_lock.h" 2

#if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS)
#  define ERTS_PROC_LOCK_OWN_IMPL 0
#else
# 44 "beam/erl_process_lock.h"
#  define ERTS_PROC_LOCK_OWN_IMPL 1
#endif
# 46 "beam/erl_process_lock.h"

#define ERTS_PROC_LOCK_ATOMIC_IMPL 0
#define ERTS_PROC_LOCK_SPINLOCK_IMPL 0
#define ERTS_PROC_LOCK_MUTEX_IMPL 0

#if !ERTS_PROC_LOCK_OWN_IMPL
#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 1
#else
# 54 "beam/erl_process_lock.h"
#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 0

#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
#  undef ERTS_PROC_LOCK_ATOMIC_IMPL
#  define ERTS_PROC_LOCK_ATOMIC_IMPL 1
#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS)
# 60 "beam/erl_process_lock.h"
#  undef ERTS_PROC_LOCK_SPINLOCK_IMPL
#  define ERTS_PROC_LOCK_SPINLOCK_IMPL 1
#else
# 63 "beam/erl_process_lock.h"
#  undef ERTS_PROC_LOCK_MUTEX_IMPL
#  define ERTS_PROC_LOCK_MUTEX_IMPL 1
#endif
# 66 "beam/erl_process_lock.h"

#endif
# 68 "beam/erl_process_lock.h"

#define ERTS_PROC_LOCK_MAX_BIT 4

typedef erts_aint32_t ErtsProcLocks;

typedef struct erts_proc_lock_t_ {
#if ERTS_PROC_LOCK_OWN_IMPL
#if ERTS_PROC_LOCK_ATOMIC_IMPL
    erts_smp_atomic32_t flags;
#else
# 78 "beam/erl_process_lock.h"
    ErtsProcLocks flags;
#endif
# 80 "beam/erl_process_lock.h"
    erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt_main;
    erts_lcnt_lock_t lcnt_link;
    erts_lcnt_lock_t lcnt_msgq;
    erts_lcnt_lock_t lcnt_btm;
    erts_lcnt_lock_t lcnt_status;
#endif
# 88 "beam/erl_process_lock.h"
#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 89 "beam/erl_process_lock.h"
    erts_mtx_t main;
    erts_mtx_t link;
    erts_mtx_t msgq;
    erts_mtx_t btm;
    erts_mtx_t status;
#else
# 95 "beam/erl_process_lock.h"
#  error "no implementation"
#endif
# 97 "beam/erl_process_lock.h"
#ifdef ERTS_PROC_LOCK_DEBUG
    erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
#endif
# 100 "beam/erl_process_lock.h"
} erts_proc_lock_t;

/* Process lock flags */

/*
 * Main lock:
 *   The main lock is held by the scheduler running a process. It
 *   is used to protect all fields in the process structure except
 *   for those fields protected by other process locks (follows).
 */
#define ERTS_PROC_LOCK_MAIN		(((ErtsProcLocks) 1) << 0)

/*
 * Link lock:
 *   Protects the following fields in the process structure:
 *   * nlinks
 *   * monitors
 *   * suspend_monitors
 */
#define ERTS_PROC_LOCK_LINK		(((ErtsProcLocks) 1) << 1)

/*
 * Message queue lock:
 *   Protects the following fields in the process structure:
 *   * msg_inq
 */
#define ERTS_PROC_LOCK_MSGQ		(((ErtsProcLocks) 1) << 2)

/*
 * Bif timer lock:
 *   Protects the following fields in the process structure:
 *   * bif_timers
 */
#define ERTS_PROC_LOCK_BTM		(((ErtsProcLocks) 1) << 3)

/*
 * Status lock:
 *   Protects the following fields in the process structure:
 *   * pending_suspenders
 *   * suspendee
 *   * ...
 */
#define ERTS_PROC_LOCK_STATUS		(((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT)

/*
 * Special fields:
 *
 *   The following fields are read only and can be read if at
 *   least one process lock (whichever one doesn't matter)
 *   is held, or if the process structure is guaranteed not to
 *   disappear by other means (e.g. pix lock is held):
 *     * id
 *
 *   The following fields are only allowed to be written if
 *   all process locks are held, and are allowed to be read if
 *   at least one process lock (whichever one doesn't matter)
 *   is held:
 *     * tracer_proc
 *     * tracer_flags
 *
 *   The following fields are only allowed to be accessed if
 *   both the schedule queue lock and at least one process lock
 *   (whichever one doesn't matter) are held:
 *     * prio
 *     * next
 *     * scheduler_flags
 */

/*
 * Other rules regarding process locking:
 *
 * Exiting processes:
 *   When changing state to exiting (ERTS_PSFLG_EXITING) on a process,
 *   you are required to take all process locks (ERTS_PROC_LOCKS_ALL).
 *   Thus, by holding at least one process lock (whichever one doesn't
 *   matter) you are guaranteed that the process won't exit until the
 *   lock you are holding has been released.
 *
 * Lock order:
 *   Process locks with low numeric values has to be locked before
 *   process locks with high numeric values. E.g., main locks has
 *   to be locked before message queue locks.
 *
 *   When process locks with the same numeric value are to be locked
 *   on multiple processes, locks on processes with low process ids
 *   have to be locked before locks on processes with high process
 *   ids. E.g., if the main and the message queue locks are to be
 *   locked on processes p1 and p2 and p1->common.id < p2->common.id,
 *   then locks should be locked in the following order:
 *     1. main lock on p1
 *     2. main lock on p2
 *     3. message queue lock on p1
 *     4. message queue lock on p2
 */

/* Other lock flags */
#define ERTS_PROC_LOCK_WAITER_SHIFT (ERTS_PROC_LOCK_MAX_BIT + 1)


/* ERTS_PROC_LOCKS_* are combinations of process locks */

#define ERTS_PROC_LOCKS_MSG_RECEIVE	(ERTS_PROC_LOCK_MSGQ		\
					 | ERTS_PROC_LOCK_STATUS)
#define ERTS_PROC_LOCKS_MSG_SEND	(ERTS_PROC_LOCK_MSGQ		\
					 | ERTS_PROC_LOCK_STATUS)
#define ERTS_PROC_LOCKS_XSIG_SEND	ERTS_PROC_LOCK_STATUS

#define ERTS_PROC_LOCKS_ALL \
  ((((ErtsProcLocks) 1) << (ERTS_PROC_LOCK_MAX_BIT + 1)) - 1)

#define ERTS_PROC_LOCKS_ALL_MINOR	(ERTS_PROC_LOCKS_ALL \
					 & ~ERTS_PROC_LOCK_MAIN)


#define ERTS_PIX_LOCKS_BITS		10
#define ERTS_NO_OF_PIX_LOCKS		(1 << ERTS_PIX_LOCKS_BITS)


#endif /* #ifndef ERTS_PROC_LOCK_TYPE__ */
# 219 "beam/erl_process_lock.h"

#ifndef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#ifndef ERTS_PROC_LOCK_LOCK_CHECK__
#define  ERTS_PROC_LOCK_LOCK_CHECK__

/* Lock counter implemetation */

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_proc_lock__(P,I,L) erts_smp_proc_lock_x__(P,I,L,__FILE__,__LINE__)
#define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__)
#endif
# 230 "beam/erl_process_lock.h"

#if defined(ERTS_SMP) && defined (ERTS_ENABLE_LOCK_COUNT)

void erts_lcnt_proc_lock_init(Process *p);
void erts_lcnt_proc_lock_destroy(Process *p);
void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks);
void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line);
void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks);
void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks);
void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res);

void erts_lcnt_enable_proc_lock_count(int enable);

#endif /* ERTS_ENABLE_LOCK_COUNT*/
# 244 "beam/erl_process_lock.h"



/* --- Process lock checking ----------------------------------------------- */

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
#define ERTS_SMP_CHK_NO_PROC_LOCKS \
  erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__)
#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \
  erts_proc_lc_chk_only_proc_main((P))
void erts_proc_lc_lock(Process *p, ErtsProcLocks locks,
		       char *file, unsigned int line);
void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
			  char *file, unsigned int line);
void erts_proc_lc_unlock(Process *p, ErtsProcLocks locks);
void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks);
void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks);
void erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks);
void erts_proc_lc_chk_only_proc_main(Process *p);
void erts_proc_lc_chk_no_proc_locks(char *file, int line);
ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p);
int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks);
void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks,
			       char* file, unsigned int line);
void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
#else
# 270 "beam/erl_process_lock.h"
#define ERTS_SMP_CHK_NO_PROC_LOCKS
#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P)
#endif
# 273 "beam/erl_process_lock.h"

#endif /* #ifndef ERTS_PROC_LOCK_LOCK_CHECK__ */
# 275 "beam/erl_process_lock.h"
#endif /* #ifndef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ */
# 276 "beam/erl_process_lock.h"

#if !defined(ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__) \
    && !defined(ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__)
#ifndef ERTS_PROCESS_LOCK_H__
#define ERTS_PROCESS_LOCK_H__

#ifdef ERTS_SMP

typedef struct {
    union {
	erts_mtx_t mtx;
	char buf[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
    } u;
} erts_pix_lock_t;

#define ERTS_PID2PIXLOCK(PID) \
    (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))])

#if ERTS_PROC_LOCK_OWN_IMPL

#if ERTS_PROC_LOCK_ATOMIC_IMPL

#define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \
  ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \
						   (erts_aint32_t) (MSK)))
#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \
  ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \
						   (erts_aint32_t) (MSK)))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \
						  (erts_aint32_t) (NEW), \
						  (erts_aint32_t) (EXPECTED)))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \
						  (erts_aint32_t) (NEW), \
						  (erts_aint32_t) (EXPECTED)))
#define ERTS_PROC_LOCK_FLGS_READ_(L) \
  ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags))

#else /* no opt atomic ops */
# 316 "beam/erl_process_lock.h"

ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *,
							ErtsProcLocks);
ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *,
						       ErtsProcLocks);
ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *,
							   ErtsProcLocks,
							   ErtsProcLocks);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsProcLocks
erts_proc_lock_flags_band(erts_proc_lock_t *lck, ErtsProcLocks mask)
{
    ErtsProcLocks res = lck->flags;
    lck->flags &= mask;
    return res;
}

ERTS_GLB_INLINE ErtsProcLocks
erts_proc_lock_flags_bor(erts_proc_lock_t *lck, ErtsProcLocks mask)
{
    ErtsProcLocks res = lck->flags;
    lck->flags |= mask;
    return res;
}

ERTS_GLB_INLINE ErtsProcLocks
erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
                             ErtsProcLocks expected)
{
    ErtsProcLocks res = lck->flags;
    if (res == expected)
        lck->flags = new;
    return res;
}

#endif
# 354 "beam/erl_process_lock.h"

#define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK))
#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
  erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
  erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
#define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags)

#endif /* end no opt atomic ops */
# 364 "beam/erl_process_lock.h"
#endif /* ERTS_PROC_LOCK_OWN_IMPL */
# 365 "beam/erl_process_lock.h"

extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];

void erts_init_proc_lock(int cpus);
void erts_proc_lock_prepare_proc_lock_waiter(void);
#if ERTS_PROC_LOCK_OWN_IMPL
void erts_proc_lock_failed(Process *,
			   erts_pix_lock_t *,
			   ErtsProcLocks,
			   ErtsProcLocks);
void erts_proc_unlock_failed(Process *,
			     erts_pix_lock_t *,
			     ErtsProcLocks);
#endif
# 379 "beam/erl_process_lock.h"

ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *);
ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *);
ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *);

ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p,
							  ErtsProcLocks locks);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *,
					    erts_pix_lock_t *,
					    ErtsProcLocks,
					    char *file, unsigned int line);
#else
# 392 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *,
					  erts_pix_lock_t *,
					  ErtsProcLocks);
#endif
# 396 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_unlock__(Process *,
					    erts_pix_lock_t *,
					    ErtsProcLocks);
ERTS_GLB_INLINE int erts_smp_proc_trylock__(Process *,
					    erts_pix_lock_t *,
					    ErtsProcLocks);

#ifdef ERTS_PROC_LOCK_DEBUG
ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int);
#endif
# 406 "beam/erl_process_lock.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck)
{
    ERTS_LC_ASSERT(pixlck);
    erts_mtx_lock(&pixlck->u.mtx);
}

ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck)
{
    ERTS_LC_ASSERT(pixlck);
    erts_mtx_unlock(&pixlck->u.mtx);
}

ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
{
    return erts_lc_mtx_is_locked(&pixlck->u.mtx);
}

/*
 * Helper function for erts_smp_proc_lock__ and erts_smp_proc_trylock__.
 *
 * Attempts to grab all of 'locks' simultaneously.
 *
 * On success, returns zero.
 *
 * On failure, returns the p->locks at the moment it tried to grab them,
 * at least some of which will intersect with 'locks', so it is nonzero.
 *
 * This assumes p's pix lock is held on entry if !ERTS_PROC_LOCK_ATOMIC_IMPL.
 * Does not release the pix lock.
 */
ERTS_GLB_INLINE ErtsProcLocks
erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
{
#if ERTS_PROC_LOCK_OWN_IMPL
    ErtsProcLocks expct_lflgs = 0;

    while (1) {
        ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock,
								expct_lflgs | locks,
								expct_lflgs);
        if (ERTS_LIKELY(lflgs == expct_lflgs)) {
            /* We successfully grabbed all locks. */
            return 0;
        }

        if (lflgs & locks) {
            /* Some locks we need are locked, give up. */
            return lflgs;
        }

        /* cmpxchg failed, try again (should be rare). */
        expct_lflgs = lflgs;
    }

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 464 "beam/erl_process_lock.h"

    if (locks & ERTS_PROC_LOCK_MAIN)
	if (erts_mtx_trylock(&p->lock.main) == EBUSY)
	    goto busy_main;
    if (locks & ERTS_PROC_LOCK_LINK)
	if (erts_mtx_trylock(&p->lock.link) == EBUSY)
	    goto busy_link;
    if (locks & ERTS_PROC_LOCK_MSGQ)
	if (erts_mtx_trylock(&p->lock.msgq) == EBUSY)
	    goto busy_msgq;
    if (locks & ERTS_PROC_LOCK_BTM)
	if (erts_mtx_trylock(&p->lock.btm) == EBUSY)
	    goto busy_btm;
    if (locks & ERTS_PROC_LOCK_STATUS)
	if (erts_mtx_trylock(&p->lock.status) == EBUSY)
	    goto busy_status;

    return 0;

busy_status:
    if (locks & ERTS_PROC_LOCK_BTM)
	erts_mtx_unlock(&p->lock.btm);
busy_btm:
    if (locks & ERTS_PROC_LOCK_MSGQ)
	erts_mtx_unlock(&p->lock.msgq);
busy_msgq:
    if (locks & ERTS_PROC_LOCK_LINK)
	erts_mtx_unlock(&p->lock.link);
busy_link:
    if (locks & ERTS_PROC_LOCK_MAIN)
	erts_mtx_unlock(&p->lock.main);
busy_main:

    return EBUSY;
#endif
# 499 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_proc_lock_x__(Process *p,
		     erts_pix_lock_t *pix_lck,
		     ErtsProcLocks locks,
		     char *file, unsigned int line)
#else
# 508 "beam/erl_process_lock.h"
erts_smp_proc_lock__(Process *p,
		     erts_pix_lock_t *pix_lck,
		     ErtsProcLocks locks)
#endif
# 512 "beam/erl_process_lock.h"
{
#if ERTS_PROC_LOCK_OWN_IMPL

    ErtsProcLocks old_lflgs;
#if !ERTS_PROC_LOCK_ATOMIC_IMPL
    erts_pix_lock(pix_lck);
#endif
# 519 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_lock(&(p->lock), locks);
#endif
# 523 "beam/erl_process_lock.h"

    ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_proc_lc_lock(p, locks, file, line);
#endif
# 529 "beam/erl_process_lock.h"

    old_lflgs = erts_smp_proc_raw_trylock__(p, locks);

    if (old_lflgs != 0) {
	/*
         * There is lock contention, so let erts_proc_lock_failed() deal
         * with it. Note that erts_proc_lock_failed() returns with
         * pix_lck unlocked.
         */
	erts_proc_lock_failed(p, pix_lck, locks, old_lflgs);
    }

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
    else {
	ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
	erts_pix_unlock(pix_lck);
    }
#endif
# 547 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line);
#endif
# 551 "beam/erl_process_lock.h"

#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 1);
#endif
# 555 "beam/erl_process_lock.h"

#if ERTS_PROC_LOCK_ATOMIC_IMPL
    ETHR_COMPILER_BARRIER;
#endif
# 559 "beam/erl_process_lock.h"

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 561 "beam/erl_process_lock.h"
    if (locks & ERTS_PROC_LOCK_MAIN)
	erts_mtx_lock(&p->lock.main);
    if (locks & ERTS_PROC_LOCK_LINK)
	erts_mtx_lock(&p->lock.link);
    if (locks & ERTS_PROC_LOCK_MSGQ)
	erts_mtx_lock(&p->lock.msgq);
    if (locks & ERTS_PROC_LOCK_BTM)
	erts_mtx_lock(&p->lock.btm);
    if (locks & ERTS_PROC_LOCK_STATUS)
	erts_mtx_lock(&p->lock.status);

#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 1);
#endif
# 575 "beam/erl_process_lock.h"

#endif
# 577 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void
erts_smp_proc_unlock__(Process *p,
		       erts_pix_lock_t *pix_lck,
		       ErtsProcLocks locks)
{
#if ERTS_PROC_LOCK_OWN_IMPL
    ErtsProcLocks old_lflgs;

#if ERTS_PROC_LOCK_ATOMIC_IMPL
    ETHR_COMPILER_BARRIER;
#endif
# 590 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_unlock(&(p->lock), locks);
#endif
# 594 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_proc_lc_unlock(p, locks);
#endif
# 598 "beam/erl_process_lock.h"
#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 0);
#endif
# 601 "beam/erl_process_lock.h"

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
    erts_pix_lock(pix_lck);
#endif
# 605 "beam/erl_process_lock.h"

    old_lflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock);

    ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);
    ERTS_LC_ASSERT(locks == (old_lflgs & locks));

    while (1) {
        /*
         * We'll atomically unlock every lock that has no waiter.
         * If any locks with waiters remain we'll let
         * erts_proc_unlock_failed() deal with them.
         */
        ErtsProcLocks wait_locks =
            (old_lflgs >> ERTS_PROC_LOCK_WAITER_SHIFT) & locks;

        /* What p->lock will look like with all non-waited locks released. */
        ErtsProcLocks want_lflgs = old_lflgs & (wait_locks | ~locks);

        if (want_lflgs != old_lflgs) {
            ErtsProcLocks new_lflgs =
                ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs);

            if (new_lflgs != old_lflgs) {
                /* cmpxchg failed, try again. */
                old_lflgs = new_lflgs;
                continue;
            }
        }

        /* We have successfully unlocked every lock with no waiter. */

        if (want_lflgs & locks) {
            /* Locks with waiters remain. */
            /* erts_proc_unlock_failed() returns with pix_lck unlocked. */
            erts_proc_unlock_failed(p, pix_lck, want_lflgs & locks);
        }
        else {
#if !ERTS_PROC_LOCK_ATOMIC_IMPL
            erts_pix_unlock(pix_lck);
#endif
# 645 "beam/erl_process_lock.h"
        }

        break;
    }

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 651 "beam/erl_process_lock.h"

#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 0);
#endif
# 655 "beam/erl_process_lock.h"

    if (locks & ERTS_PROC_LOCK_STATUS)
	erts_mtx_unlock(&p->lock.status);
    if (locks & ERTS_PROC_LOCK_BTM)
	erts_mtx_unlock(&p->lock.btm);
    if (locks & ERTS_PROC_LOCK_MSGQ)
	erts_mtx_unlock(&p->lock.msgq);
    if (locks & ERTS_PROC_LOCK_LINK)
	erts_mtx_unlock(&p->lock.link);
    if (locks & ERTS_PROC_LOCK_MAIN)
	erts_mtx_unlock(&p->lock.main);
#endif
# 667 "beam/erl_process_lock.h"

}

ERTS_GLB_INLINE int
erts_smp_proc_trylock__(Process *p,
			erts_pix_lock_t *pix_lck,
			ErtsProcLocks locks)
{
#if ERTS_PROC_LOCK_OWN_IMPL
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);
    if (erts_proc_lc_trylock_force_busy(p, locks)) {
	res = EBUSY; /* Make sure caller can handle the situation without
			causing a lock order violation to occur */
    }
    else
#endif
# 686 "beam/erl_process_lock.h"
    {

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
	erts_pix_lock(pix_lck);
#endif
# 691 "beam/erl_process_lock.h"

	if (erts_smp_proc_raw_trylock__(p, locks) != 0) {
	    /* Didn't get all locks... */
	    res = EBUSY;

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
	    erts_pix_unlock(pix_lck);
#endif
# 699 "beam/erl_process_lock.h"
	}
	else {
	    res = 0;

	    ERTS_LC_ASSERT(locks
			   == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
	    erts_pix_unlock(pix_lck);
#endif
# 709 "beam/erl_process_lock.h"

#ifdef ERTS_PROC_LOCK_DEBUG
	    erts_proc_lock_op_debug(p, locks, 1);
#endif
# 713 "beam/erl_process_lock.h"
	}
    }
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_trylock(&(p->lock), locks, res);
#endif
# 718 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_proc_lc_trylock(p, locks, res == 0, __FILE__, __LINE__);
#endif
# 722 "beam/erl_process_lock.h"

#if ERTS_PROC_LOCK_ATOMIC_IMPL
    ETHR_COMPILER_BARRIER;
#endif
# 726 "beam/erl_process_lock.h"
    return res;

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 729 "beam/erl_process_lock.h"
    if (erts_smp_proc_raw_trylock__(p, locks) != 0)
	return EBUSY;
    else {
#ifdef ERTS_PROC_LOCK_DEBUG
	erts_proc_lock_op_debug(p, locks, 1);
#endif
# 735 "beam/erl_process_lock.h"
	return 0;
    }
#endif
# 738 "beam/erl_process_lock.h"
}

#ifdef ERTS_PROC_LOCK_DEBUG
ERTS_GLB_INLINE void 
erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
{
    int i;
    for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) {
	ErtsProcLocks lock = ((ErtsProcLocks) 1) << i;
	if (locks & lock) {
	    erts_aint32_t lock_count;
	    if (locked) {
		lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]);
		ERTS_LC_ASSERT(lock_count == 1);
	    }
	    else {
		lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]);
		ERTS_LC_ASSERT(lock_count == 0);
	    }
	}
    }
}
#endif
# 761 "beam/erl_process_lock.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 763 "beam/erl_process_lock.h"

#endif /* ERTS_SMP */
# 765 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line);
#else
# 769 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks);
#endif
# 771 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks);
ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_inc_refc(Process *);
ERTS_GLB_INLINE void erts_proc_dec_refc(Process *);
ERTS_GLB_INLINE void erts_proc_add_refc(Process *, Sint);
ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
#else
# 785 "beam/erl_process_lock.h"
erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
#endif 
# 787 "beam/erl_process_lock.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_smp_proc_lock_x__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
			 NULL,
#else
# 793 "beam/erl_process_lock.h"
			 ERTS_PID2PIXLOCK(p->common.id),
#endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
# 795 "beam/erl_process_lock.h"
			 locks, file, line);
#elif defined(ERTS_SMP)
# 797 "beam/erl_process_lock.h"
    erts_smp_proc_lock__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
			 NULL,
#else
# 801 "beam/erl_process_lock.h"
			 ERTS_PID2PIXLOCK(p->common.id),
#endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
# 803 "beam/erl_process_lock.h"
			 locks);
#endif /*ERTS_SMP*/
# 805 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void
erts_smp_proc_unlock(Process *p, ErtsProcLocks locks)
{
#ifdef ERTS_SMP
    erts_smp_proc_unlock__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
			   NULL,
#else
# 815 "beam/erl_process_lock.h"
			   ERTS_PID2PIXLOCK(p->common.id),
#endif
# 817 "beam/erl_process_lock.h"
			   locks);
#endif
# 819 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE int
erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
{
#ifndef ERTS_SMP
    return 0;
#else
# 827 "beam/erl_process_lock.h"
    return erts_smp_proc_trylock__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
				   NULL,
#else
# 831 "beam/erl_process_lock.h"
				   ERTS_PID2PIXLOCK(p->common.id),
#endif
# 833 "beam/erl_process_lock.h"
				   locks);
#endif
# 835 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p)
{
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    erts_ptab_atmc_inc_refc(&p->common);
#else
# 843 "beam/erl_process_lock.h"
    erts_ptab_inc_refc(&p->common);
#endif
# 845 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
{
    Sint referred;
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    referred = erts_ptab_atmc_dec_test_refc(&p->common);
#else
# 854 "beam/erl_process_lock.h"
    referred = erts_ptab_dec_test_refc(&p->common);
#endif
# 856 "beam/erl_process_lock.h"
    if (!referred) {
	ASSERT(ERTS_PROC_IS_EXITING(p));
	erts_free_proc(p);
    }
}

ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
{
    Sint referred;
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc);
#else
# 869 "beam/erl_process_lock.h"
    referred = erts_ptab_add_test_refc(&p->common, add_refc);
#endif
# 871 "beam/erl_process_lock.h"
    if (!referred) {
	ASSERT(ERTS_PROC_IS_EXITING(p));
	erts_free_proc(p);
    }
}

ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p)
{
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    return erts_ptab_atmc_read_refc(&p->common);
#else
# 883 "beam/erl_process_lock.h"
    return erts_ptab_read_refc(&p->common);
#endif
# 885 "beam/erl_process_lock.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 888 "beam/erl_process_lock.h"

#ifdef ERTS_SMP
void erts_proc_lock_init(Process *);
void erts_proc_lock_fin(Process *);
void erts_proc_safelock(Process *a_proc,
			ErtsProcLocks a_have_locks,
			ErtsProcLocks a_need_locks,
			Process *b_proc,
			ErtsProcLocks b_have_locks,
			ErtsProcLocks b_need_locks);
#endif
# 899 "beam/erl_process_lock.h"

/*
 * --- Process table lookup ------------------------------------------------
 *
 * erts_pid2proc() and friends looks up the process structure of a pid
 * and at the same time acquires process locks in the smp case. Locks
 * on currently executing process and looked up process are taken according
 * to the lock order, i.e., locks on currently executing process may have
 * been released and reacquired.
 *
 * erts_pid2proc_opt() currently accepts the following flags:
 *   ERTS_P2P_FLG_ALLOW_OTHER_X    Lookup process even if it currently
 *                                 is exiting.
 */

#define ERTS_P2P_FLG_ALLOW_OTHER_X	(1 <<  0)
#define ERTS_P2P_FLG_TRY_LOCK		(1 <<  1)
#define ERTS_P2P_FLG_INC_REFC		(1 <<  2)

#define ERTS_PROC_LOCK_BUSY ((Process *) &erts_invalid_process)

#define erts_pid2proc(PROC, HL, PID, NL) \
  erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0)


ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid);

#ifndef ERTS_SMP
ERTS_GLB_INLINE
#endif
# 931 "beam/erl_process_lock.h"
Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Process *erts_pix2proc(int ix)
{
    Process *proc;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_proc));
    proc = (Process *) erts_ptab_pix2intptr_nob(&erts_proc, ix);
    return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc;
}

ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid)
{
    Process *proc;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_pid(pid))
	return NULL;

    proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
						 internal_pid_index(pid));
    if (proc && proc->common.id != pid)
	return NULL;
    return proc;
}

ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid)
{
    Process *proc = erts_proc_lookup_raw(pid);
    if (proc && ERTS_PROC_IS_EXITING(proc))
	return NULL;
    return proc;
}

#ifndef ERTS_SMP
ERTS_GLB_INLINE Process *
erts_pid2proc_opt(Process *c_p_unused,
		  ErtsProcLocks c_p_have_locks_unused,
		  Eterm pid,
		  ErtsProcLocks pid_need_locks_unused,
		  int flags)
{
    Process *proc = erts_proc_lookup_raw(pid);
    if (!proc)
	return NULL;
    if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
	&& ERTS_PROC_IS_EXITING(proc))
	return NULL;
    if (flags & ERTS_P2P_FLG_INC_REFC)
	erts_proc_inc_refc(proc);
    return proc;
}
#endif /* !ERTS_SMP */
# 986 "beam/erl_process_lock.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 988 "beam/erl_process_lock.h"

#endif /* #ifndef ERTS_PROCESS_LOCK_H__ */
# 990 "beam/erl_process_lock.h"
#endif /* #if !defined(ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__) 
	  && !defined(ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__) */
# 992 "beam/erl_process_lock.h"
# 44 "beam/erl_process.h" 2
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 1 "beam/erl_port.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_PORT_TYPE__
#define ERL_PORT_TYPE__
typedef struct _erl_drv_port Port;
typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
#endif
# 26 "beam/erl_port.h"

#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__)
#define ERL_PORT_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_port_task.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port.h"
# 31 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_ptab.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_port.h"
# 32 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port.h"
# 33 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_port.h"
# 34 "beam/erl_port.h"

#ifndef __WIN32__
#define ERTS_DEFAULT_MAX_PORTS (1 << 16)
#else
# 38 "beam/erl_port.h"
/* 
 *  Do not default to as many max ports on Windows
 *  as there are no os limits to stop system
 *  from running amok. If allowed to go too high
 *  windows rarely recovers from the errors and
 *  other OS processes can be effected. 
 */
#define ERTS_DEFAULT_MAX_PORTS (1 << 13)
#endif /* __WIN32__ */
# 47 "beam/erl_port.h"
#define ERTS_MIN_PORTS 1024

extern int erts_port_synchronous_ops;
extern int erts_port_schedule_all_ops;
extern int erts_port_parallelism;

typedef struct erts_driver_t_ erts_driver_t;

/*
 * It would have been preferred to use NULL as value of
 * ERTS_INVALID_ERL_DRV_PORT. That would, however, not be
 * backward compatible. In pre-R16 systems, 0 was a valid
 * port handle and -1 was used as invalid handle, so we
 * are stuck with it.
 */
#define ERTS_INVALID_ERL_DRV_PORT ((struct _erl_drv_port *) ((SWord) -1))
#ifdef DEBUG
/* Make sure we use this api, and do not cast directly */
#define ERTS_ErlDrvPort2Port(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((Port *) ((PH) - 4711)))
#define ERTS_Port2ErlDrvPort(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((ErlDrvPort) ((PH) + 4711)))
#else
# 74 "beam/erl_port.h"
#define ERTS_ErlDrvPort2Port(PH) ((Port *) (PH))
#define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH))
#endif
# 77 "beam/erl_port.h"

#define SMALL_IO_QUEUE 5   /* Number of fixed elements */

typedef struct {
    ErlDrvSizeT size;       /* total size in bytes */

    SysIOVec* v_start;
    SysIOVec* v_end;
    SysIOVec* v_head;
    SysIOVec* v_tail;
    SysIOVec  v_small[SMALL_IO_QUEUE];

    ErlDrvBinary** b_start;
    ErlDrvBinary** b_end;
    ErlDrvBinary** b_head;
    ErlDrvBinary** b_tail;
    ErlDrvBinary*  b_small[SMALL_IO_QUEUE];
} ErlIOQueue;

typedef struct line_buf {  /* Buffer used in line oriented I/O */
    ErlDrvSizeT bufsiz;      /* Size of character buffer */
    ErlDrvSizeT ovlen;       /* Length of overflow data */
    ErlDrvSizeT ovsiz;       /* Actual size of overflow buffer */
    char data[1];            /* Starting point of buffer data,
			      data[0] is a flag indicating an unprocess CR,
			      The rest is the overflow buffer. */
} LineBuf;

/*
 * Items part of erlang:port_info/1 result. Note am_registered_name
 * *need* to be first.
 */

#define ERTS_PORT_INFO_1_ITEMS				\
    {	am_registered_name,	/* Needs to be first */	\
	am_name,					\
	am_links,					\
	am_id,						\
	am_connected,					\
	am_input,					\
	am_output,					\
	am_os_pid }

/*
 * Port Specific Data.
 *
 * Only use PrtSD for very rarely used data.
 */

#define ERTS_PRTSD_SCHED_ID 0

#define ERTS_PRTSD_SIZE 1

typedef struct {
    void *data[ERTS_PRTSD_SIZE];
} ErtsPrtSD;

#ifdef ERTS_SMP
typedef struct ErtsXPortsList_ ErtsXPortsList;
#endif
# 137 "beam/erl_port.h"

/*
 * Port locking:
 *
 * Locking is done either driver specific or port specific. When
 * driver specific locking is used, all instances of the driver,
 * i.e. ports running the driver, share the same lock. When port
 * specific locking is used each instance have its own lock.
 *
 * Most fields in the Port structure are protected by the lock
 * referred to by the 'lock' field. This lock is shared between
 * all ports running the same driver when driver specific locking
 * is used.
 *
 * The 'sched' field is protected by the run queue lock that the
 * port currently is assigned to.
 *
 */

struct _erl_drv_port {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    ErtsPortTaskSched sched;
    ErtsPortTaskHandle timeout_task;
#ifdef ERTS_SMP
    erts_mtx_t *lock;
    ErtsXPortsList *xports;
    erts_smp_atomic_t run_queue;
#else
# 166 "beam/erl_port.h"
    erts_atomic32_t refc;
    int cleanup;
#endif
# 169 "beam/erl_port.h"
    erts_atomic_t connected;	/* A connected process */
    Eterm caller;		/* Current caller. */
    erts_smp_atomic_t data;	/* Data associated with port. */
    Uint bytes_in;		/* Number of bytes read */
    Uint bytes_out;		/* Number of bytes written */

    ErlIOQueue ioq;              /* driver accessible i/o queue */
    DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
    char *name;		         /* String used in the open */
    erts_driver_t* drv_ptr;
    UWord drv_data;
    SWord os_pid;                /* Child process ID */
    ErtsProcList *suspended;	 /* List of suspended processes. */
    LineBuf *linebuf;            /* Buffer to hold data not ready for
				    process to get (line oriented I/O)*/
    erts_atomic32_t state;	 /* Status and type flags */
    int control_flags;		 /* Flags for port_control()  */
    ErlDrvPDL port_data_lock;

    ErtsPrtSD *psd;		 /* Port specific data */
    int reds; /* Only used while executing driver callbacks */
};


void erts_init_port_data(Port *);
void erts_cleanup_port_data(Port *);
Uint erts_port_data_size(Port *);
ErlOffHeap *erts_port_data_offheap(Port *);

#define ERTS_PORT_GET_CONNECTED(PRT) \
    ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
    erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
    erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))


struct erl_drv_port_data_lock {
    erts_mtx_t mtx;
    erts_atomic_t refc;
    Port *prt;
};

ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsRunQueue *
erts_port_runq(Port *prt)
{
#ifdef ERTS_SMP
    ErtsRunQueue *rq1, *rq2;
    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
    if (!rq1)
	return NULL;
    while (1) {
	erts_smp_runq_lock(rq1);
	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
	if (rq1 == rq2)
	    return rq1;
	erts_smp_runq_unlock(rq1);
	rq1 = rq2;
	if (!rq1)
	    return NULL;
    }
#else
# 235 "beam/erl_port.h"
    return ERTS_RUNQ_IX(0);
#endif
# 237 "beam/erl_port.h"
}

#endif
# 240 "beam/erl_port.h"


ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix);
ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_prtsd_get(Port *prt, int ix)
{
    return prt->psd ? prt->psd->data[ix] : NULL;
}

ERTS_GLB_INLINE void *
erts_prtsd_set(Port *prt, int ix, void *data)
{
    if (prt->psd) {
	void *old = prt->psd->data[ix];
	prt->psd->data[ix] = data;
	return old;
    }
    else {
	prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
	prt->psd->data[ix] = data;
	return NULL;
    }
}

#endif
# 269 "beam/erl_port.h"

Eterm erts_request_io_bytes(Process *c_p);


/* port status flags */

#define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
/* Port have begun exiting */
#define ERTS_PORT_SFLG_EXITING		((Uint32) (1 <<  1))
/* Distribution port */
#define ERTS_PORT_SFLG_DISTRIBUTION	((Uint32) (1 <<  2))
#define ERTS_PORT_SFLG_BINARY_IO	((Uint32) (1 <<  3))
#define ERTS_PORT_SFLG_SOFT_EOF		((Uint32) (1 <<  4))
/* Flow control */
/* Port is closing (no i/o accepted) */
#define ERTS_PORT_SFLG_CLOSING		((Uint32) (1 <<  5))
/* Send a closed message when terminating */
#define ERTS_PORT_SFLG_SEND_CLOSED	((Uint32) (1 <<  6))
/* Line orinted io on port */  
#define ERTS_PORT_SFLG_LINEBUF_IO	((Uint32) (1 <<  7))
/* Immortal port (only certain system ports) */
#define ERTS_PORT_SFLG_FREE		((Uint32) (1 <<  8))
#define ERTS_PORT_SFLG_INITIALIZING	((Uint32) (1 <<  9))
/* Port uses port specific locking (opposed to driver specific locking) */
#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 10))
#define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 11))
/* Last port to terminate halts the emulator */
#define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 12))
#ifdef DEBUG
/* Only debug: make sure all flags aren't cleared unintentionally */
#define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
#endif
# 301 "beam/erl_port.h"

/* Combinations of port status flags */ 
#define ERTS_PORT_SFLGS_DEAD						\
  (ERTS_PORT_SFLG_FREE | ERTS_PORT_SFLG_INITIALIZING)
#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
  (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID)
#define ERTS_PORT_SFLGS_INVALID_LOOKUP					\
  (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
   | ERTS_PORT_SFLG_EXITING						\
   | ERTS_PORT_SFLG_CLOSING)
#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP				\
  (ERTS_PORT_SFLGS_INVALID_LOOKUP					\
   | ERTS_PORT_SFLG_DISTRIBUTION)

/*
 * Costs in reductions for some port operations.
 */
#define ERTS_PORT_REDS_EXECUTE		(CONTEXT_REDS/4)
#define ERTS_PORT_REDS_FREE		(CONTEXT_REDS/400)
#define ERTS_PORT_REDS_TIMEOUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_INPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_OUTPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EVENT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUTV	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUT	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EXIT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CONNECT		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_UNLINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_LINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_BADSIG		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_CONTROL		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CALL		(CONTEXT_REDS/50)
#define ERTS_PORT_REDS_INFO		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_TERMINATE	(CONTEXT_REDS/50)

void print_port_info(Port *, int, void *);
void erts_port_free(Port *);
#ifndef ERTS_SMP
void erts_port_cleanup(Port *);
#endif
# 341 "beam/erl_port.h"
void erts_fire_port_monitor(Port *prt, Eterm ref);
#ifdef ERTS_SMP
int erts_port_handle_xports(Port *);
#endif
# 345 "beam/erl_port.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_is_port_locked(Port *);
#endif
# 349 "beam/erl_port.h"

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);

ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt)
{
    erts_ptab_atmc_inc_refc(&prt->common);
}

ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt)
{
    int referred = erts_ptab_atmc_dec_test_refc(&prt->common);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc)
{
    int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
{
    return erts_ptab_atmc_read_refc(&prt->common);
}

ERTS_GLB_INLINE int
erts_smp_port_trylock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    return erts_mtx_trylock(prt->lock);
#else
# 393 "beam/erl_port.h"
    return 0;
#endif
# 395 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_lock(prt->lock);
#endif
# 405 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_unlock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_unlock(prt->lock);
#endif
# 415 "beam/erl_port.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 418 "beam/erl_port.h"


#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS)			\
    (!(PP)							\
     || (erts_atomic32_read_nob(&(PP)->state) & (FLGS))	\
     || (PP)->common.id != (ID))

/* port lookup */

#define INVALID_PORT(PP, ID) \
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP)

/* Invalidate trace port if anything suspicious, for instance
 * that the port is a distribution port or it is busy.
 */
#define INVALID_TRACER_PORT(PP, ID)					\
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)

#define ERTS_PORT_SCHED_ID(P, ID) \
  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))

extern const Port erts_invalid_port;
#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)

int erts_is_port_ioq_empty(Port *);
void erts_terminate_port(Port *);

#ifdef ERTS_SMP
Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
#endif
# 448 "beam/erl_port.h"

ERTS_GLB_INLINE Port *erts_pix2port(int);
ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm);
ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32);
ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
ERTS_GLB_INLINE void erts_port_release(Port *);
#ifdef ERTS_SMP
ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
#endif
# 459 "beam/erl_port.h"
ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int);
ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *);
ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort);
ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm);
ERTS_GLB_INLINE int erts_is_port_alive(Eterm);
ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm);
ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *);

#define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL)

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Port *erts_pix2port(int ix)
{
    Port *prt;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_port));
    prt = (Port *) erts_ptab_pix2intptr_nob(&erts_port, ix);
    return prt == ERTS_PORT_LOCK_BUSY ? NULL : prt;
}

ERTS_GLB_INLINE Port *
erts_port_lookup_raw(Eterm id)
{
    Port *prt;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));
    return prt && prt->common.id == id ? prt : NULL;
}

ERTS_GLB_INLINE Port *
erts_port_lookup(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt = erts_port_lookup_raw(id);
    return (!prt
	    ? NULL
	    : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state))
	       ? NULL
	       : prt));
}


ERTS_GLB_INLINE Port*
erts_id2port(Eterm id)
{
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

    erts_smp_port_lock(prt);
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) {
	erts_smp_port_unlock(prt);
	return NULL;
    }

    return prt;
}


ERTS_GLB_INLINE Port*
erts_id2port_sflgs(Eterm id,
		   Process *c_p, ErtsProcLocks c_p_locks,
		   Uint32 invalid_sflgs)
{
#ifdef ERTS_SMP
    int no_proc_locks = !c_p || !c_p_locks;
#endif
# 543 "beam/erl_port.h"
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

#ifdef ERTS_SMP
    if (no_proc_locks)
	erts_smp_port_lock(prt);
    else if (erts_smp_port_trylock(prt) == EBUSY) {
	/* Unlock process locks, and acquire locks in lock order... */
	erts_smp_proc_unlock(c_p, c_p_locks);
	erts_smp_port_lock(prt);
	erts_smp_proc_lock(c_p, c_p_locks);
    }
#endif
# 568 "beam/erl_port.h"
    state = erts_atomic32_read_nob(&prt->state);
    if (state & invalid_sflgs) {
#ifdef ERTS_SMP
	erts_smp_port_unlock(prt);
#endif
# 573 "beam/erl_port.h"
	return NULL;
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_port_release(Port *prt)
{
    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
#ifdef ERTS_SMP
    erts_smp_port_unlock(prt);
#else
# 587 "beam/erl_port.h"
    if (prt->cleanup) {
	prt->cleanup = 0;
	erts_port_cleanup(prt);
    }
#endif
# 592 "beam/erl_port.h"
}

#ifdef ERTS_SMP

/*
 * erts_thr_id2port_sflgs() and erts_thr_port_release() can
 * be used by unmanaged threads in the SMP case.
 */
ERTS_GLB_INLINE Port *
erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt;
    ErtsThrPrgrDelayHandle dhndl;

    if (is_not_internal_port(id))
	return NULL;

    dhndl = erts_thr_progress_unmanaged_delay();

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id) {
	erts_thr_progress_unmanaged_continue(dhndl);
	prt = NULL;
    }
    else {
	erts_aint32_t state;
	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
	    erts_port_inc_refc(prt);
	    erts_thr_progress_unmanaged_continue(dhndl);
	}

	erts_mtx_lock(prt->lock);
	state = erts_atomic32_read_nob(&prt->state);
	if (state & invalid_sflgs) {
	    erts_mtx_unlock(prt->lock);
	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
		erts_port_dec_refc(prt);
	    prt = NULL;
	}
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_thr_port_release(Port *prt)
{
    erts_mtx_unlock(prt->lock);
#ifdef ERTS_SMP
    if (!erts_thr_progress_is_managed_thread())
	erts_port_dec_refc(prt);
#endif
# 646 "beam/erl_port.h"
}

#endif
# 649 "beam/erl_port.h"

ERTS_GLB_INLINE Port *
erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    ASSERT(prt != NULL);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;

    if (lock_pdl && prt->port_data_lock)
	driver_pdl_lock(prt->port_data_lock);

#if ERTS_ENABLE_LOCK_CHECK
    if (!ERTS_IS_CRASH_DUMPING) {
	if (erts_lc_is_emu_thr()) {
	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	    ERTS_LC_ASSERT(!prt->port_data_lock
			   || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
	else {
	    ERTS_LC_ASSERT(prt->port_data_lock);
	    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
    }
#endif
# 674 "beam/erl_port.h"

    if (erts_atomic32_read_nob(&prt->state)
	& ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
	if (lock_pdl && prt->port_data_lock)
	    driver_pdl_unlock(prt->port_data_lock);
	return ERTS_INVALID_ERL_DRV_PORT;
    }
    return prt;
}

ERTS_GLB_INLINE Port *
erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    erts_aint32_t state;
    ASSERT(prt);
    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;
    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
		       || ERTS_IS_CRASH_DUMPING);
    /* 
     * This state check is only needed since a driver callback
     * might terminate the port, and then call back into the
     * emulator. Drivers should preferably have been forbidden
     * to call into the emulator after terminating the port,
     * but it has been like this for ages. Perhaps forbid this
     * in some future major release?
     */
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
	return ERTS_INVALID_ERL_DRV_PORT;
    if (statep)
	*statep = state;
    return prt;
}

ERTS_GLB_INLINE Eterm
erts_drvport2id(ErlDrvPort drvport)
{
    Port *prt = erts_drvport2port(drvport);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return am_undefined;
    else
	return prt->common.id;
}

ERTS_GLB_INLINE Uint32
erts_portid2status(Eterm id)
{
    Port *prt = erts_port_lookup_raw(id);
    if (prt)
	return (Uint32) erts_atomic32_read_acqb(&prt->state);
    else
	return ERTS_PORT_SFLG_INVALID;
}

ERTS_GLB_INLINE int
erts_is_port_alive(Eterm id)
{
    return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID
				       | ERTS_PORT_SFLGS_DEAD));
}

ERTS_GLB_INLINE int
erts_is_valid_tracer_port(Eterm id)
{
    return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
}

ERTS_GLB_INLINE int
erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
{
    int reds = 0;
    erts_aint32_t state;

    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));

    state = erts_atomic32_read_nob(&prt->state);
    if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) {
	reds += ERTS_PORT_REDS_TERMINATE;
	erts_terminate_port(prt);
	state = erts_atomic32_read_nob(&prt->state);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
    }

#ifdef ERTS_SMP
    if (prt->xports) {
	reds += erts_port_handle_xports(prt);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	ASSERT(!prt->xports);
    }
#endif
# 767 "beam/erl_port.h"

    if (statep)
	*statep = state;

    return reds;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 775 "beam/erl_port.h"

void erts_port_resume_procs(Port *);

struct binary;

#define ERTS_P2P_SIG_TYPE_BAD			0
#define ERTS_P2P_SIG_TYPE_OUTPUT		1
#define ERTS_P2P_SIG_TYPE_OUTPUTV		2
#define ERTS_P2P_SIG_TYPE_CONNECT		3
#define ERTS_P2P_SIG_TYPE_EXIT			4
#define ERTS_P2P_SIG_TYPE_CONTROL		5
#define ERTS_P2P_SIG_TYPE_CALL			6
#define ERTS_P2P_SIG_TYPE_INFO			7
#define ERTS_P2P_SIG_TYPE_LINK			8
#define ERTS_P2P_SIG_TYPE_UNLINK		9

#define ERTS_P2P_SIG_TYPE_BITS			4
#define ERTS_P2P_SIG_TYPE_MASK \
    ((1 << ERTS_P2P_SIG_TYPE_BITS) - 1)

#define ERTS_P2P_SIG_DATA_FLG(N) \
    (1 << (ERTS_P2P_SIG_TYPE_BITS + (N)))
#define ERTS_P2P_SIG_DATA_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG(0)
#define ERTS_P2P_SIG_DATA_FLG_REPLY		ERTS_P2P_SIG_DATA_FLG(1)
#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG(2)
#define ERTS_P2P_SIG_DATA_FLG_FORCE		ERTS_P2P_SIG_DATA_FLG(3)
#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT	ERTS_P2P_SIG_DATA_FLG(4)
#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK	ERTS_P2P_SIG_DATA_FLG(5)
#define ERTS_P2P_SIG_DATA_FLG_SCHED		ERTS_P2P_SIG_DATA_FLG(6)
#define ERTS_P2P_SIG_DATA_FLG_ASYNC		ERTS_P2P_SIG_DATA_FLG(7)

struct ErtsProc2PortSigData_ {
    int flags;
    Eterm caller;
    Uint32 ref[ERTS_MAX_REF_NUMBERS];
    union {
	struct {
	    Eterm from;
	    ErlIOVec *evp;
	    ErlDrvBinary *cbinp;
	} outputv;
	struct {
	    Eterm from;
	    char *bufp;
	    ErlDrvSizeT size;
	} output;
	struct {
	    Eterm from;
	    Eterm connected;
	} connect;
	struct {
	    Eterm from;
	    Eterm reason;
	    ErlHeapFragment *bp;
	} exit;
	struct {
	    struct binary *binp;
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} control;
	struct {
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} call;
	struct {
	    Eterm item;
	} info;
	struct {
	    Eterm port;
	    Eterm to;
	} link;
	struct {
	    Eterm from;
	} unlink;
    } u;
} ;

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp);
ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return !0;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return !0;
    default:				return 0;
    }
}

ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return sigdp->u.output.size;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return sigdp->u.outputv.evp->size;
    default:				return (ErlDrvSizeT) 0;
    }
}

#endif
# 882 "beam/erl_port.h"

#define ERTS_PROC2PORT_SIG_EXEC			0
#define ERTS_PROC2PORT_SIG_ABORT		1
#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND	2
#define ERTS_PROC2PORT_SIG_ABORT_CLOSED		3

typedef int (*ErtsProc2PortSigCallback)(Port *,
					erts_aint32_t,
					int,
					ErtsProc2PortSigData *);

typedef enum {
    ERTS_PORT_OP_BADARG,
    ERTS_PORT_OP_CALLER_EXIT,
    ERTS_PORT_OP_BUSY,
    ERTS_PORT_OP_BUSY_SCHEDULED,
    ERTS_PORT_OP_SCHEDULED,
    ERTS_PORT_OP_DROPPED,
    ERTS_PORT_OP_DONE
} ErtsPortOpResult;

ErtsPortOpResult
erts_schedule_proc2port_signal(Process *,
			       Port *,
			       Eterm,
			       Eterm *,
			       ErtsProc2PortSigData *,
			       int,
			       ErtsPortTaskHandle *,
			       ErtsProc2PortSigCallback);

int erts_deliver_port_exit(Port *, Eterm, Eterm, int);

/*
 * Port signal flags
 */
#define ERTS_PORT_SIG_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG_BANG_OP
#define ERTS_PORT_SIG_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG_NOSUSPEND
#define ERTS_PORT_SIG_FLG_FORCE			ERTS_P2P_SIG_DATA_FLG_FORCE
#define ERTS_PORT_SIG_FLG_BROKEN_LINK		ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK
#define ERTS_PORT_SIG_FLG_BAD_OUTPUT		ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT
#define ERTS_PORT_SIG_FLG_FORCE_SCHED		ERTS_P2P_SIG_DATA_FLG_SCHED
#define ERTS_PORT_SIG_FLG_ASYNC			ERTS_P2P_SIG_DATA_FLG_ASYNC
/* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */
#define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL	ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT

/*
 * Port ! {Owner, {command, Data}}
 * Port ! {Owner, {connect, NewOwner}}
 * Port ! {Owner, close}
 */
ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);

/*
 * Signals from processes to ports.
 */
ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);

#endif
# 948 "beam/erl_port.h"
# 48 "beam/erl_process.h" 2
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 29 "beam/erl_smp.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 51 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 1 "beam/erl_message.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1997-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERL_MESSAGE_H__
#define __ERL_MESSAGE_H__

struct proc_bin;
struct external_thing_;

/*
 * This struct represents data that must be updated by structure copy,
 * but is stored outside of any heap.
 */

struct erl_off_heap_header {
    Eterm thing_word;
    Uint size;
#if HALFWORD_HEAP
    void* dummy_ptr_padding__;
#endif
# 38 "beam/erl_message.h"
    struct erl_off_heap_header* next;
};

#define OH_OVERHEAD(oh, size) do { \
    (oh)->overhead += size;        \
} while(0)

typedef struct erl_off_heap {
    struct erl_off_heap_header* first;
    Uint64 overhead;     /* Administrative overhead (used to force GC). */
} ErlOffHeap;

#define ERTS_INIT_OFF_HEAP(OHP)			\
    do {					\
	(OHP)->first = NULL;			\
	(OHP)->overhead = 0;			\
    } while (0)

typedef struct {
    enum {
        FACTORY_CLOSED = 0,
        FACTORY_HALLOC,
        FACTORY_HEAP_FRAGS,
        FACTORY_STATIC,
        FACTORY_TMP
    } mode;
    Process* p;
    Eterm* hp_start;
    Eterm* hp;
    Eterm* hp_end;
    struct erl_heap_fragment* heap_frags;
    struct erl_heap_fragment* heap_frags_saved;
    ErlOffHeap* off_heap;
    ErlOffHeap off_heap_saved;
    Uint32 alloc_type;
} ErtsHeapFactory;

void erts_factory_proc_init(ErtsHeapFactory*, Process*);
void erts_factory_proc_prealloc_init(ErtsHeapFactory*, Process*, Sint size);
void erts_factory_message_init(ErtsHeapFactory*, Process*, Eterm* hp, struct erl_heap_fragment*);
void erts_factory_static_init(ErtsHeapFactory*, Eterm* hp, Uint size, ErlOffHeap*);
void erts_factory_tmp_init(ErtsHeapFactory*, Eterm* hp, Uint size, Uint32 atype);
void erts_factory_dummy_init(ErtsHeapFactory*);

Eterm* erts_produce_heap(ErtsHeapFactory*, Uint need, Uint xtra);
Eterm* erts_reserve_heap(ErtsHeapFactory*, Uint need);
void erts_factory_close(ErtsHeapFactory*);
void erts_factory_trim_and_close(ErtsHeapFactory*,Eterm *brefs, Uint brefs_size);
void erts_factory_undo(ErtsHeapFactory*);

#ifdef CHECK_FOR_HOLES
# define ERTS_FACTORY_HOLE_CHECK(f) do {    \
        /*if ((f)->p) erts_check_for_holes((f)->p);*/ \
    } while (0)
#else
# 93 "beam/erl_message.h"
# define ERTS_FACTORY_HOLE_CHECK(p)
#endif
# 95 "beam/erl_message.h"

#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 96 "beam/erl_message.h"
# 1 "beam/external.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Same order as the ordering of terms in erlang */

/* Since there are 255 different External tag values to choose from
   There is no reason to not be extravagant.
   Hence, the different tags for large/small tuple e.t.c
*/


#ifdef ERTS_WANT_EXTERNAL_TAGS
#ifndef ERTS_EXTERNAL_TAGS
#define ERTS_EXTERNAL_TAGS

#define SMALL_INTEGER_EXT 'a'
#define INTEGER_EXT       'b'
#define FLOAT_EXT         'c'
#define ATOM_EXT          'd'
#define SMALL_ATOM_EXT    's'
#define REFERENCE_EXT     'e'
#define NEW_REFERENCE_EXT 'r'
#define PORT_EXT          'f'
#define NEW_FLOAT_EXT     'F'
#define PID_EXT           'g'
#define SMALL_TUPLE_EXT   'h'
#define LARGE_TUPLE_EXT   'i'
#define NIL_EXT           'j'
#define STRING_EXT        'k'
#define LIST_EXT          'l'
#define BINARY_EXT        'm'
#define BIT_BINARY_EXT    'M'
#define SMALL_BIG_EXT     'n'
#define LARGE_BIG_EXT     'o'
#define NEW_FUN_EXT       'p'
#define EXPORT_EXT        'q'
#define MAP_EXT           't'
#define FUN_EXT           'u'
#define ATOM_UTF8_EXT     'v'
#define SMALL_ATOM_UTF8_EXT 'w'

#define DIST_HEADER       'D'
#define ATOM_CACHE_REF    'R'
#define ATOM_INTERNAL_REF2 'I'
#define ATOM_INTERNAL_REF3 'K'
#define BINARY_INTERNAL_REF 'J'
#define BIT_BINARY_INTERNAL_REF 'L'
#define COMPRESSED        'P'

#if 0
/* Not used anymore */
#define CACHED_ATOM       'C'
#define NEW_CACHE         'N'
#endif
# 72 "beam/external.h"


#define VERSION_MAGIC 131   /* 130 in erlang 4.2 */
                /* Increment this when changing the external format. */
                /* ON the other hand, don't change the external format */
                /* since that breaks other people's code! */

#endif /* ERTS_EXTERNAL_TAGS */
# 80 "beam/external.h"
#endif /* ERTS_WANT_EXTERNAL_TAGS */
# 81 "beam/external.h"

#ifndef ERL_EXTERNAL_H__
#define ERL_EXTERNAL_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 85 "beam/external.h"
# 1 "beam/erl_node_tables.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_NODE_TABLES_H__
#define ERL_NODE_TABLES_H__

/*
 * The "node_tables module" contain two (hash) tables: the node_table
 * and the dist_table.
 *
 * The elements of the node_table represents a specific incarnation of
 * an Erlang node and has {Nodename, Creation} pairs as keys. Elements
 * in the node_table are referred to from node containers (see
 * node_container_utils.h).
 *
 * The elements of the dist_table represents a (potential) connection
 * to an Erlang node and has Nodename as key. Elements in the
 * dist_table are either referred to from elements in the node_table
 * or from the process or port structure of the entity controlling
 * the connection.
 *
 * Both tables are garbage collected by reference counting.
 */

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 42 "beam/erl_node_tables.h"
# 43 "beam/erl_node_tables.h"
#if 0 /* expanded by -frewrite-includes */
#include "hash.h"
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_node_tables.h"
# 44 "beam/erl_node_tables.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 44 "beam/erl_node_tables.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 45 "beam/erl_node_tables.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 45 "beam/erl_node_tables.h"
# 46 "beam/erl_node_tables.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 46 "beam/erl_node_tables.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 29 "beam/erl_smp.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 47 "beam/erl_node_tables.h" 2
#define ERTS_PORT_TASK_ONLY_BASIC_TYPES__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port_task.h"
#endif /* expanded by -frewrite-includes */
# 48 "beam/erl_node_tables.h"
# 1 "beam/erl_port_task.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2006-2013. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
 * Description:	Scheduling of port tasks
 *
 * Author: 	Rickard Green
 */

#ifndef ERTS_PORT_TASK_H_BASIC_TYPES__
#define ERTS_PORT_TASK_H_BASIC_TYPES__
#if 0 /* expanded by -frewrite-includes */
#include "erl_sys_driver.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/erl_port_task.h"
# 1 "beam/erl_sys_driver.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2001-2013. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
 * Include file for erlang driver writers.
 */

#ifndef __ERL_SYS_DRIVER_H__
#define __ERL_SYS_DRIVER_H__

#ifdef __ERL_DRIVER_H__
#error erl_sys_driver.h cannot be included after erl_driver.h
#endif
# 31 "beam/erl_sys_driver.h"

#define ERL_SYS_DRV

typedef long ErlDrvEvent; /* An event to be selected on. */

/* typedef struct _SysDriverOpts SysDriverOpts; defined in sys.h */

#if 0 /* expanded by -frewrite-includes */
#include "erl_driver.h"
#endif /* expanded by -frewrite-includes */
# 38 "beam/erl_sys_driver.h"
# 1 "beam/erl_driver.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1999-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Include file for erlang driver writers.
 */

#ifndef __ERL_DRIVER_H__
#define __ERL_DRIVER_H__

#ifdef HAVE_CONFIG_H
#if 0 /* expanded by -frewrite-includes */
#  include "config.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/erl_driver.h"
# 1 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h" 1
/* armv6-portbld-freebsd11.0/config.h.  Generated by configure.  */
/* config.h.in.  Generated from configure.in by autoheader.  */


#define GHBN_R_SOLARIS  2
#define GHBN_R_AIX      3
#define GHBN_R_GLIBC    4


/* Assumed cache-line size (in bytes) */
#define ASSUMED_CACHE_LINE_SIZE 64

/* Define the brk() argument type. */
#define BRK_ARG_TYPE const void *

/* Define the brk() return type. */
#define BRK_RET_TYPE int

/* Modern style mcontext_t in MacOSX */
/* #undef DARWIN_MODERN_MCONTEXT */

/* Define if you need to include rpc/types.h to get INADDR_LOOPBACK defined */
/* #undef DEF_INADDR_LOOPBACK_IN_RPC_TYPES_H */

/* Define if you need to include winsock2.h to get INADDR_LOOPBACK defined */
/* #undef DEF_INADDR_LOOPBACK_IN_WINSOCK2_H */

/* Define if you want to disable child waiter thread */
/* #undef DISABLE_CHILD_WAITER_THREAD */

/* Define if you want to disable vfork. */
/* #undef DISABLE_VFORK */

/* Define to 1 if your processor stores the words in a double in middle-endian
   format (like some ARMs). */
/* #undef DOUBLE_MIDDLE_ENDIAN */

/* Define if you want to enable child waiter thread */
/* #undef ENABLE_CHILD_WAITER_THREAD */

/* Dirty scheduler support */
/* #undef ERL_DRV_DIRTY_SCHEDULER_SUPPORT */

/* Dirty scheduler support */
/* #undef ERL_NIF_DIRTY_SCHEDULER_SUPPORT */

/* Define if sbrk()/brk() wrappers can track malloc()s core memory use */
/* #undef ERTS_BRK_WRAPPERS_CAN_TRACK_MALLOC */

/* Define to override the default number of write_concurrency locks */
/* #undef ERTS_DB_HASH_LOCK_CNT */

/* Define if the emulator supports dirty schedulers */
/* #undef ERTS_DIRTY_SCHEDULERS */

/* The only reason ERTS_EMU_CMDLINE_FLAGS exists is to force modification of
   config.h when the emulator command line flags are modified by configure */
#define ERTS_EMU_CMDLINE_FLAGS " -O -pipe -mfloat-abi=softfp -fno-strict-aliasing -I/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0   -g  -I/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0    -DUSE_THREADS -D_THREAD_SAFE -D_REENTRANT -DPOSIX_THREADS  -Werror=return-type  -Wall -Wstrict-prototypes -Wmissing-prototypes -Wdeclaration-after-statement"

/* Define if you have kernel poll and want to use it */
#define ERTS_ENABLE_KERNEL_POLL 1

/* Define if OS monotonic clock is corrected */
/* #undef ERTS_HAVE_CORRECTED_OS_MONOTONIC_TIME */

/* Define if you have a low resolution OS monotonic clock */
/* #undef ERTS_HAVE_LOW_RESOLUTION_OS_MONOTONIC_LOW */

/* Define if the smp emulator is built */
#define ERTS_HAVE_SMP_EMU 1

/* Define if dlopen() needs to be called before first call to dlerror() */
/* #undef ERTS_NEED_DLOPEN_BEFORE_DLERROR */

/* Save compile time? */
#define ERTS_SAVED_COMPILE_TIME 1

/* Define if poll() should be used instead of select() */
#define ERTS_USE_POLL 1

/* Define if __after_morecore_hook can track malloc()s core memory use. */
/* #undef ERTS___AFTER_MORECORE_HOOK_CAN_TRACK_MALLOC */

/* Define if bigendian */
/* #undef ETHR_BIGENDIAN */

/* Define if gcc wont let you clobber ebx with cmpxchg8b and position
   independent code */
/* #undef ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX */

/* Define if you get a register shortage with cmpxchg8b and position
   independent code */
/* #undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE */

/* Define if you want to disable native ethread implementations */
/* #undef ETHR_DISABLE_NATIVE_IMPLS */

/* Define if you want to force usage of pthread rwlocks */
/* #undef ETHR_FORCE_PTHREAD_RWLOCK */

/* Define if you use a gcc that supports the double word cmpxchg instruction
   */
/* #undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */

/* Define if you use a gcc that supports -msse2 and understand sse2 specific
   asm statements */
/* #undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT */

/* Define if you have a clock_gettime() with a monotonic clock */
#define ETHR_HAVE_CLOCK_GETTIME_MONOTONIC 1

/* Define if you have all ethread defines */
#define ETHR_HAVE_ETHREAD_DEFINES 1

/* Define as a boolean indicating whether you have a gcc compatible compiler
   capable of generating the ARM DMB instruction, and are compiling for an ARM
   processor with ARM DMB instruction support, or not */
#define ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION 0

/* Define as a boolean indicating whether you have a gcc __atomic builtins or
   not */
#define ETHR_HAVE_GCC___ATOMIC_BUILTINS 1

/* Define if you have a monotonic gethrtime() */
/* #undef ETHR_HAVE_GETHRTIME */

/* Define if you have libatomic_ops atomic operations */
/* #undef ETHR_HAVE_LIBATOMIC_OPS */

/* Define if you have a linux futex implementation. */
/* #undef ETHR_HAVE_LINUX_FUTEX */

/* Define if you have a mach clock_get_time() with a monotonic clock */
/* #undef ETHR_HAVE_MACH_CLOCK_GET_TIME */

/* Define if the pthread.h header file is in pthread/mit directory. */
/* #undef ETHR_HAVE_MIT_PTHREAD_H */

/* Define if you have the pthread_attr_setguardsize function. */
#define ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE 1

/* Define if pthread_cond_timedwait() can be used with a monotonic clock */
#define ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC 1

/* Define if you have ibm style pthread_getname_np */
/* #undef ETHR_HAVE_PTHREAD_GETNAME_NP_2 */

/* Define if you have linux style pthread_getname_np */
/* #undef ETHR_HAVE_PTHREAD_GETNAME_NP_3 */

/* Define if you have the <pthread.h> header file. */
#define ETHR_HAVE_PTHREAD_H 1

/* Define if you have the pthread_rwlockattr_setkind_np() function. */
/* #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP */

/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
   attribute. */
/* #undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP */

/* Define if you have darwin style pthread_setname_np */
/* #undef ETHR_HAVE_PTHREAD_SETNAME_NP_1 */

/* Define if you have linux style pthread_setname_np */
/* #undef ETHR_HAVE_PTHREAD_SETNAME_NP_2 */

/* Define if you have bsd style pthread_set_name_np */
/* #undef ETHR_HAVE_PTHREAD_SET_NAME_NP_2 */

/* Define if you have the pthread_spin_lock function. */
#define ETHR_HAVE_PTHREAD_SPIN_LOCK 1

/* Define if you have the pthread_yield() function. */
#define ETHR_HAVE_PTHREAD_YIELD 1

/* Define if you have the <sched.h> header file. */
#define ETHR_HAVE_SCHED_H 1

/* Define if you have the sched_yield() function. */
#define ETHR_HAVE_SCHED_YIELD 1

/* Define if you have the <sys/time.h> header file. */
#define ETHR_HAVE_SYS_TIME_H 1

/* Define if you have _InterlockedAnd() */
/* #undef ETHR_HAVE__INTERLOCKEDAND */

/* Define if you have _InterlockedAnd64() */
/* #undef ETHR_HAVE__INTERLOCKEDAND64 */

/* Define if you have _InterlockedCompareExchange() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE */

/* Define if you have _InterlockedCompareExchange128() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128 */

/* Define if you have _InterlockedCompareExchange64() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 */

/* Define if you have _InterlockedCompareExchange64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ */

/* Define if you have _InterlockedCompareExchange64_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL */

/* Define if you have _InterlockedCompareExchange_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ */

/* Define if you have _InterlockedCompareExchange_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL */

/* Define if you have _InterlockedDecrement() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT */

/* Define if you have _InterlockedDecrement64() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT64 */

/* Define if you have _InterlockedDecrement64_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL */

/* Define if you have _InterlockedDecrement_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL */

/* Define if you have _InterlockedExchange() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGE */

/* Define if you have _InterlockedExchange64() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 */

/* Define if you have _InterlockedExchangeAdd() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD */

/* Define if you have _InterlockedExchangeAdd64() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 */

/* Define if you have _InterlockedExchangeAdd64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ */

/* Define if you have _InterlockedExchangeAdd_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ */

/* Define if you have _InterlockedIncrement() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT */

/* Define if you have _InterlockedIncrement64() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT64 */

/* Define if you have _InterlockedIncrement64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ */

/* Define if you have _InterlockedIncrement_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ */

/* Define if you have _InterlockedOr() */
/* #undef ETHR_HAVE__INTERLOCKEDOR */

/* Define if you have _InterlockedOr64() */
/* #undef ETHR_HAVE__INTERLOCKEDOR64 */

/* Define as a bitmask corresponding to the word sizes that
   __atomic_add_fetch() can handle on your system */
#define ETHR_HAVE___atomic_add_fetch 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_compare_exchange_n() can handle on your system */
#define ETHR_HAVE___atomic_compare_exchange_n 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_fetch_and() can handle on your system */
#define ETHR_HAVE___atomic_fetch_and 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_fetch_or() can handle on your system */
#define ETHR_HAVE___atomic_fetch_or 12

/* Define as a bitmask corresponding to the word sizes that __atomic_load_n()
   can handle on your system */
#define ETHR_HAVE___atomic_load_n 12

/* Define as a bitmask corresponding to the word sizes that __atomic_store_n()
   can handle on your system */
#define ETHR_HAVE___atomic_store_n 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_add_and_fetch() can handle on your system */
#define ETHR_HAVE___sync_add_and_fetch 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_fetch_and_and() can handle on your system */
#define ETHR_HAVE___sync_fetch_and_and 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_fetch_and_or() can handle on your system */
#define ETHR_HAVE___sync_fetch_and_or 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_synchronize() can handle on your system */
#define ETHR_HAVE___sync_synchronize ~0

/* Define as a bitmask corresponding to the word sizes that
   __sync_val_compare_and_swap() can handle on your system */
#define ETHR_HAVE___sync_val_compare_and_swap 12

/* Define if you want to modify the default stack size */
/* #undef ETHR_MODIFIED_DEFAULT_STACK_SIZE */

/* Define to the monotonic clock id to use */
#define ETHR_MONOTONIC_CLOCK_ID CLOCK_MONOTONIC

/* Define if you need the <nptl/pthread.h> header file. */
/* #undef ETHR_NEED_NPTL_PTHREAD_H */

/* Define if you have OSE style threads */
/* #undef ETHR_OSE_THREADS */

/* Define if you prefer gcc native ethread implementations */
/* #undef ETHR_PREFER_GCC_NATIVE_IMPLS */

/* Define if you prefer libatomic_ops native ethread implementations */
/* #undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS */

/* Define if you have pthreads */
#define ETHR_PTHREADS 1

/* Define if pthread_yield() returns an int. */
/* #undef ETHR_PTHREAD_YIELD_RET_INT */

/* Define if sched_yield() returns an int. */
#define ETHR_SCHED_YIELD_RET_INT 1

/* Define to the size of AO_t if libatomic_ops is used */
/* #undef ETHR_SIZEOF_AO_T */

/* Define to the size of int */
#define ETHR_SIZEOF_INT 4

/* Define to the size of long */
#define ETHR_SIZEOF_LONG 4

/* Define to the size of long long */
#define ETHR_SIZEOF_LONG_LONG 8

/* Define to the size of pointers */
#define ETHR_SIZEOF_PTR 4

/* Define to the size of __int128_t */
#define ETHR_SIZEOF___INT128_T 0

/* Define to the size of __int64 */
#define ETHR_SIZEOF___INT64 0

/* Define if you want to enable check for native ethread implementations */
#define ETHR_SMP_REQUIRE_NATIVE_IMPLS 1

/* Define if only run in Sparc PSO, or TSO mode */
/* #undef ETHR_SPARC_PSO */

/* Define if run in Sparc RMO, PSO, or TSO mode */
/* #undef ETHR_SPARC_RMO */

/* Define if only run in Sparc TSO mode */
/* #undef ETHR_SPARC_TSO */

/* Define if you can safely include both <sys/time.h> and <time.h>. */
#define ETHR_TIME_WITH_SYS_TIME 1

/* Define as a boolean indicating whether you trust gcc's __atomic_* builtins
   memory barrier implementations, or not */
#define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS 0

/* Define if you have win32 threads */
/* #undef ETHR_WIN32_THREADS */

/* Define if x86/x86_64 out of order instructions should be synchronized */
/* #undef ETHR_X86_OUT_OF_ORDER */

/* Define if building a halfword-heap 64bit emulator */
/* #undef HALFWORD_HEAP_EMULATOR */

/* Define to 1 if you have the <arpa/nameser.h> header file. */
#define HAVE_ARPA_NAMESER_H 1

/* Define to 1 if you have the `brk' function. */
#define HAVE_BRK 1

/* Define to 1 if you have the `clock_getres' function. */
#define HAVE_CLOCK_GETRES 1

/* define if clock_gettime() works for getting process time */
/* #undef HAVE_CLOCK_GETTIME_CPU_TIME */

/* Define if you have clock_gettime(CLOCK_MONOTONIC_RAW, _) */
/* #undef HAVE_CLOCK_GETTIME_MONOTONIC_RAW */

/* Define to 1 if you have the `clock_get_attributes' function. */
/* #undef HAVE_CLOCK_GET_ATTRIBUTES */

/* Define to 1 if you have the `closefrom' function. */
#define HAVE_CLOSEFROM 1

/* Define if you have a decl of fread that conflicts with int fread */
#define HAVE_CONFLICTING_FREAD_DECLARATION 1

/* Define if you have a putenv() that stores a copy of the key-value pair */
/* #undef HAVE_COPYING_PUTENV */

/* Define if you have cpuset_getaffinity/cpuset_setaffinity */
#define HAVE_CPUSET_xETAFFINITY 1

/* Define to 1 if you have the declaration of `getrlimit', and to 0 if you
   don't. */
#define HAVE_DECL_GETRLIMIT 1

/* Define to 1 if you have the declaration of `IN6ADDR_ANY_INIT', and to 0 if
   you don't. */
#define HAVE_DECL_IN6ADDR_ANY_INIT 1

/* Define to 1 if you have the declaration of `IN6ADDR_LOOPBACK_INIT', and to
   0 if you don't. */
#define HAVE_DECL_IN6ADDR_LOOPBACK_INIT 1

/* Define to 1 if you have the declaration of `IPV6_V6ONLY', and to 0 if you
   don't. */
#define HAVE_DECL_IPV6_V6ONLY 1

/* Define to 1 if you have the declaration of `posix2time', and to 0 if you
   don't. */
#define HAVE_DECL_POSIX2TIME 0

/* Define to 1 if you have the declaration of `RLIMIT_STACK', and to 0 if you
   don't. */
#define HAVE_DECL_RLIMIT_STACK 1

/* Define to 1 if you have the declaration of `SCTPS_BOUND', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_BOUND 0

/* Define to 1 if you have the declaration of `SCTPS_COOKIE_ECHOED', and to 0
   if you don't. */
#define HAVE_DECL_SCTPS_COOKIE_ECHOED 0

/* Define to 1 if you have the declaration of `SCTPS_COOKIE_WAIT', and to 0 if
   you don't. */
#define HAVE_DECL_SCTPS_COOKIE_WAIT 0

/* Define to 1 if you have the declaration of `SCTPS_ESTABLISHED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTPS_ESTABLISHED 0

/* Define to 1 if you have the declaration of `SCTPS_IDLE', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_IDLE 0

/* Define to 1 if you have the declaration of `SCTPS_LISTEN', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_LISTEN 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_ACK_SENT', and
   to 0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_ACK_SENT 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_PENDING', and to
   0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_PENDING 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_RECEIVED', and
   to 0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_RECEIVED 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_SENT', and to 0
   if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_SENT 0

/* Define to 1 if you have the declaration of `SCTP_ABORT', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_ABORT 1

/* Define to 1 if you have the declaration of `SCTP_ADDR_CONFIRMED', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_ADDR_CONFIRMED 1

/* Define to 1 if you have the declaration of `SCTP_ADDR_OVER', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_ADDR_OVER 1

/* Define to 1 if you have the declaration of `SCTP_BOUND', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_BOUND 1

/* Define to 1 if you have the declaration of `SCTP_CLOSED', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_CLOSED 1

/* Define to 1 if you have the declaration of `SCTP_COOKIE_ECHOED', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_COOKIE_ECHOED 1

/* Define to 1 if you have the declaration of `SCTP_COOKIE_WAIT', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_COOKIE_WAIT 1

/* Define to 1 if you have the declaration of `SCTP_DELAYED_ACK_TIME', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_DELAYED_ACK_TIME 0

/* Define to 1 if you have the declaration of `SCTP_EMPTY', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_EMPTY 0

/* Define to 1 if you have the declaration of `SCTP_EOF', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_EOF 1

/* Define to 1 if you have the declaration of `SCTP_ESTABLISHED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_ESTABLISHED 1

/* Define to 1 if you have the declaration of `SCTP_LISTEN', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_LISTEN 1

/* Define to 1 if you have the declaration of `SCTP_SENDALL', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_SENDALL 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_ACK_SENT', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_ACK_SENT 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_PENDING', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_PENDING 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_RECEIVED', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_RECEIVED 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_SENT', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_SENT 1

/* Define to 1 if you have the declaration of `SCTP_UNCONFIRMED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_UNCONFIRMED 1

/* Define to 1 if you have the declaration of `SCTP_UNORDERED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_UNORDERED 1

/* Define to 1 if you have the declaration of `setrlimit', and to 0 if you
   don't. */
#define HAVE_DECL_SETRLIMIT 1

/* Define to 1 if you have the declaration of `time2posix', and to 0 if you
   don't. */
#define HAVE_DECL_TIME2POSIX 0

/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
   */
#define HAVE_DIRENT_H 1

/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1

/* Define to 1 if you have the `dlopen' function. */
#define HAVE_DLOPEN 1

/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
/* #undef HAVE_DOPRNT */

/* Define if you have the 'end' symbol */
#define HAVE_END_SYMBOL 1

/* Define if you have a working fallocate() */
/* #undef HAVE_FALLOCATE */

/* Define to 1 if you have the <fcntl.h> header file. */
#define HAVE_FCNTL_H 1

/* Define to 1 if you have the `fdatasync' function. */
/* #undef HAVE_FDATASYNC */

/* Define to 1 if you have the `finite' function. */
#define HAVE_FINITE 1

/* Define to 1 if you have the `flockfile' function. */
#define HAVE_FLOCKFILE 1

/* Define to 1 if you have the `fork' function. */
#define HAVE_FORK 1

/* Define to 1 if you have the `fpsetmask' function. */
#define HAVE_FPSETMASK 1

/* Define to 1 if you have the `fstat' function. */
#define HAVE_FSTAT 1

/* Define if you have fwrite_unlocked */
/* #undef HAVE_FWRITE_UNLOCKED */

/* Define to 1 if you have a good `getaddrinfo' function. */
#define HAVE_GETADDRINFO 1

/* Define to 1 if you have the `gethostbyname2' function. */
#define HAVE_GETHOSTBYNAME2 1

/* Define to flavour of gethostbyname_r */
/* #undef HAVE_GETHOSTBYNAME_R */

/* Define to 1 if you have the `gethrtime' function. */
/* #undef HAVE_GETHRTIME */

/* define if gethrvtime() works and uses ioctl() to /proc/self */
/* #undef HAVE_GETHRVTIME_PROCFS_IOCTL */

/* Define to 1 if you have the `getifaddrs' function. */
#define HAVE_GETIFADDRS 1

/* Define to 1 if you have the `getipnodebyaddr' function. */
#define HAVE_GETIPNODEBYADDR 1

/* Define to 1 if you have the `getipnodebyname' function. */
#define HAVE_GETIPNODEBYNAME 1

/* Define to 1 if you have a good `getnameinfo' function. */
#define HAVE_GETNAMEINFO 1

/* Define to 1 if you have the `gettimeofday' function. */
#define HAVE_GETTIMEOFDAY 1

/* Define to 1 if you have the `gmtime_r' function. */
#define HAVE_GMTIME_R 1

/* Define to 1 if you have the <ieeefp.h> header file. */
#define HAVE_IEEEFP_H 1

/* Define to 1 if you have the `ieee_handler' function. */
/* #undef HAVE_IEEE_HANDLER */

/* Define to 1 if you have the <ifaddrs.h> header file. */
#define HAVE_IFADDRS_H 1

/* Define if ipv6 is present */
#define HAVE_IN6 1

/* Define to 1 if you have the variable in6addr_any declared. */
#define HAVE_IN6ADDR_ANY 1

/* Define to 1 if you have the variable in6addr_loopback declared. */
#define HAVE_IN6ADDR_LOOPBACK 1

/* Define to 1 if you have the `inet_pton' function. */
#define HAVE_INET_PTON 1

/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1

/* Early linux used in_addr6 instead of in6_addr, define if you have this */
/* #undef HAVE_IN_ADDR6_STRUCT */

/* Define to 1 if you have the `isfinite' function. */
/* #undef HAVE_ISFINITE */

/* Define to 1 if you have the `isinf' function. */
#define HAVE_ISINF 1

/* Define to 1 if you have the `isnan' function. */
#define HAVE_ISNAN 1

/* Define if you have kstat */
/* #undef HAVE_KSTAT */

/* Define to 1 if you have the <langinfo.h> header file. */
#define HAVE_LANGINFO_H 1

/* Define to 1 if you have the `dl' library (-ldl). */
/* #undef HAVE_LIBDL */

/* Define to 1 if you have the `dlpi' library (-ldlpi). */
/* #undef HAVE_LIBDLPI */

/* Define to 1 if you have the <libdlpi.h> header file. */
/* #undef HAVE_LIBDLPI_H */

/* Define to 1 if you have the `inet' library (-linet). */
/* #undef HAVE_LIBINET */

/* Define to 1 if you have the `m' library (-lm). */
#define HAVE_LIBM 1

/* Define to 1 if you have the `util' library (-lutil). */
#define HAVE_LIBUTIL 1

/* Define to 1 if you have the `z' library (-lz). */
#define HAVE_LIBZ 1

/* Define to 1 if you have the <limits.h> header file. */
#define HAVE_LIMITS_H 1

/* Define to 1 if you have the <linux/falloc.h> header file. */
/* #undef HAVE_LINUX_FALLOC_H */

/* Define to 1 if you have the `localtime_r' function. */
#define HAVE_LOCALTIME_R 1

/* Define to 1 if you have the `log2' function. */
#define HAVE_LOG2 1

/* Define to 1 if you have the <malloc.h> header file. */
/* #undef HAVE_MALLOC_H */

/* Define to 1 if you have the `mallopt' function. */
/* #undef HAVE_MALLOPT */

/* Define to 1 if you have the `memcpy' function. */
#define HAVE_MEMCPY 1

/* Define to 1 if you have the `memmove' function. */
#define HAVE_MEMMOVE 1

/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1

/* Define if the pthread.h header file is in pthread/mit directory. */
/* #undef HAVE_MIT_PTHREAD_H */

/* Define to 1 if you have the `mlockall' function. */
#define HAVE_MLOCKALL 1

/* Define to 1 if you have the `mmap' function. */
#define HAVE_MMAP 1

/* Define if you have a monotonic erts_os_hrtime() implementation */
/* #undef HAVE_MONOTONIC_ERTS_SYS_HRTIME */

/* Define to 1 if you have the `mremap' function. */
/* #undef HAVE_MREMAP */

/* Define if setsockopt() accepts multicast options */
#define HAVE_MULTICAST_SUPPORT 1

/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
/* #undef HAVE_NDIR_H */

/* Define to 1 if you have the <netpacket/packet.h> header file. */
/* #undef HAVE_NETPACKET_PACKET_H */

/* Define to 1 if you have the <net/errno.h> header file. */
/* #undef HAVE_NET_ERRNO_H */

/* Define to 1 if you have the <net/if_dl.h> header file. */
#define HAVE_NET_IF_DL_H 1

/* Define to 1 if you have the `nl_langinfo' function. */
#define HAVE_NL_LANGINFO 1

/* Define if you don't have a definition of INADDR_LOOPBACK */
/* #undef HAVE_NO_INADDR_LOOPBACK */

/* Define to 1 if you have the `openpty' function. */
#define HAVE_OPENPTY 1

/* Define if you have the "ose_spi/ose_spi.h" header file. */
/* #undef HAVE_OSE_SPI_H */

/* Define to 1 if you have the `poll' function. */
#define HAVE_POLL 1

/* Define to 1 if you have the <poll.h> header file. */
#define HAVE_POLL_H 1

/* Define to 1 if you have the `posix2time' function. */
#define HAVE_POSIX2TIME 1

/* Define to 1 if you have the `posix_fadvise' function. */
#define HAVE_POSIX_FADVISE 1

/* Define if you have a working posix_fallocate() */
#define HAVE_POSIX_FALLOCATE 

/* Define to 1 if you have the `posix_memalign' function. */
#define HAVE_POSIX_MEMALIGN 1

/* Define to 1 if you have the `ppoll' function. */
#define HAVE_PPOLL 1

/* Define to 1 if you have the `pread' function. */
#define HAVE_PREAD 1

/* Define if you have processor_bind functionality */
/* #undef HAVE_PROCESSOR_BIND */

/* Define if you have pset functionality */
/* #undef HAVE_PSET */

/* Define if you have the <pthread.h> header file. */
#define HAVE_PTHREAD_H 1

/* Define to 1 if you have the <pty.h> header file. */
/* #undef HAVE_PTY_H */

/* Define if you have putc_unlocked */
#define HAVE_PUTC_UNLOCKED 1

/* Define to 1 if you have the `pwrite' function. */
#define HAVE_PWRITE 1

/* Define to 1 if you have the `res_gethostbyname' function. */
/* #undef HAVE_RES_GETHOSTBYNAME */

/* Define to 1 if you have the `sbrk' function. */
#define HAVE_SBRK 1

/* Define to 1 if you have the <sched.h> header file. */
#define HAVE_SCHED_H 1

/* Define if you have sched_getaffinity/sched_setaffinity */
/* #undef HAVE_SCHED_xETAFFINITY */

/* Define to 1 if you have the `sctp_bindx' function. */
#define HAVE_SCTP_BINDX 1

/* Define to 1 if you have the `sctp_freeladdrs' function. */
#define HAVE_SCTP_FREELADDRS 1

/* Define to 1 if you have the `sctp_freepaddrs' function. */
#define HAVE_SCTP_FREEPADDRS 1

/* Define to 1 if you have the `sctp_getladdrs' function. */
#define HAVE_SCTP_GETLADDRS 1

/* Define to 1 if you have the `sctp_getpaddrs' function. */
#define HAVE_SCTP_GETPADDRS 1

/* Define to 1 if you have the <netinet/sctp.h> header file */
#define HAVE_SCTP_H 1

/* Define to 1 if you have the `sctp_peeloff' function. */
#define HAVE_SCTP_PEELOFF 1

/* Define to 1 if you have the <sdkddkver.h> header file. */
/* #undef HAVE_SDKDDKVER_H */

/* Define to 1 if you have the `sendfile' function. */
#define HAVE_SENDFILE 1

/* Define to 1 if you have the `sendfilev' function. */
/* #undef HAVE_SENDFILEV */

/* Define to 1 if you have the `setlocale' function. */
#define HAVE_SETLOCALE 1

/* Define to 1 if you have the `setns' function. */
/* #undef HAVE_SETNS */

/* Define to 1 if you have the <setns.h> header file. */
/* #undef HAVE_SETNS_H */

/* Define to 1 if you have the `setsid' function. */
#define HAVE_SETSID 1

/* Define if we have socklen_t */
#define HAVE_SOCKLEN_T 1

/* define if you have the Solaris/ultrasparc /dev/perfmon interface */
/* #undef HAVE_SOLARIS_SPARC_PERFMON */

/* Define if you have SO_BSDCOMPAT flag on sockets */
/* #undef HAVE_SO_BSDCOMPAT */

/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1

/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1

/* Define to 1 if you have the `strerror' function. */
#define HAVE_STRERROR 1

/* Define to 1 if you have the `strerror_r' function. */
#define HAVE_STRERROR_R 1

/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1

/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1

/* Define to 1 if you have the `strlcat' function. */
#define HAVE_STRLCAT 1

/* Define to 1 if you have the `strlcpy' function. */
#define HAVE_STRLCPY 1

/* Define to 1 if you have the `strncasecmp' function. */
#define HAVE_STRNCASECMP 1

/* Define to 1 if `ifr_enaddr' is member of `struct ifreq'. */
/* #undef HAVE_STRUCT_IFREQ_IFR_ENADDR */

/* Define to 1 if `ifr_hwaddr' is member of `struct ifreq'. */
/* #undef HAVE_STRUCT_IFREQ_IFR_HWADDR */

/* Define to 1 if `spp_flags' is member of `struct sctp_paddrparams'. */
#define HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS 1

/* Define to 1 if `spp_pathmtu' is member of `struct sctp_paddrparams'. */
#define HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_PATHMTU 1

/* Define to 1 if `spp_sackdelay' is member of `struct sctp_paddrparams'. */
/* #undef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY */

/* Define to 1 if `sre_data' is member of `struct sctp_remote_error'. */
#define HAVE_STRUCT_SCTP_REMOTE_ERROR_SRE_DATA 1

/* Define to 1 if `ssf_data' is member of `struct sctp_send_failed'. */
#define HAVE_STRUCT_SCTP_SEND_FAILED_SSF_DATA 1

/* Define to 1 if you have the <syslog.h> header file. */
#define HAVE_SYSLOG_H 1

/* Define if you have systemd daemon */
/* #undef HAVE_SYSTEMD_DAEMON */

/* Define to 1 if you have the <systemd/sd-daemon.h> header file. */
/* #undef HAVE_SYSTEMD_SD_DAEMON_H */

/* Define if you have <sys/devpoll.h> header file. */
/* #undef HAVE_SYS_DEVPOLL_H */

/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
   */
/* #undef HAVE_SYS_DIR_H */

/* Define if you have the <sys/epoll.h> header file. */
/* #undef HAVE_SYS_EPOLL_H */

/* Define if you have <sys/event.h> header file. */
#define HAVE_SYS_EVENT_H 1

/* Define to 1 if you have the <sys/ioctl.h> header file. */
#define HAVE_SYS_IOCTL_H 1

/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
   */
/* #undef HAVE_SYS_NDIR_H */

/* Define to 1 if you have the <sys/resource.h> header file */
#define HAVE_SYS_RESOURCE_H 1

/* Define to 1 if you have the <sys/sdt.h> header file. */
/* #undef HAVE_SYS_SDT_H */

/* Define to 1 if you have the <sys/socketio.h> header file. */
/* #undef HAVE_SYS_SOCKETIO_H */

/* Define to 1 if you have the <sys/socket.h> header file. */
#define HAVE_SYS_SOCKET_H 1

/* Define to 1 if you have the <sys/sockio.h> header file. */
#define HAVE_SYS_SOCKIO_H 1

/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1

/* Define to 1 if you have the <sys/stropts.h> header file. */
/* #undef HAVE_SYS_STROPTS_H */

/* Define to 1 if you have the <sys/sysctl.h> header file. */
#define HAVE_SYS_SYSCTL_H 1

/* Define to 1 if you have the <sys/timerfd.h> header file. */
/* #undef HAVE_SYS_TIMERFD_H */

/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1

/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1

/* Define to 1 if you have the <sys/uio.h> header file. */
#define HAVE_SYS_UIO_H 1

/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */
#define HAVE_SYS_WAIT_H 1

/* Define if termcap functions exists */
#define HAVE_TERMCAP 1

/* Define to 1 if you have the `time2posix' function. */
#define HAVE_TIME2POSIX 1

/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1

/* Define to 1 if you have the <util.h> header file. */
/* #undef HAVE_UTIL_H */

/* Define to 1 if you have the <utmp.h> header file. */
/* #undef HAVE_UTMP_H */

/* Define to 1 if you have the `vfork' function. */
#define HAVE_VFORK 1

/* Define to 1 if you have the <vfork.h> header file. */
/* #undef HAVE_VFORK_H */

/* Define to 1 if you have the `vprintf' function. */
#define HAVE_VPRINTF 1

/* Define to 1 if you have a `wcwidth' function. */
#define HAVE_WCWIDTH 1

/* Define to 1 if you have the <windows.h> header file. */
/* #undef HAVE_WINDOWS_H */

/* Define to 1 if you have the <winsock2.h> header file. */
/* #undef HAVE_WINSOCK2_H */

/* Define to 1 if `fork' works. */
#define HAVE_WORKING_FORK 1

/* Define if you have a working posix_openpt implementation */
#define HAVE_WORKING_POSIX_OPENPT 1

/* Define to 1 if `vfork' works. */
#define HAVE_WORKING_VFORK 1

/* Define to 1 if you have the `writev' function. */
#define HAVE_WRITEV 1

/* Define to 1 if you have the <ws2tcpip.h> header file. */
/* #undef HAVE_WS2TCPIP_H */

/* Define to 1 if you have the `_brk' function. */
#define HAVE__BRK 1

/* Define if you have the '_end' symbol */
#define HAVE__END_SYMBOL 1

/* Define to 1 if you have the `_sbrk' function. */
#define HAVE__SBRK 1

/* Define to 1 if you have the `__brk' function. */
/* #undef HAVE___BRK */

/* Define to 1 if you have the `__sbrk' function. */
/* #undef HAVE___SBRK */

/* Define to enable HiPE */
#define HIPE 1

/* Define to monotonic clock id to use */
#define HRTIME_CLOCK_ID CLOCK_UPTIME_PRECISE

/* Define as a string of monotonic clock id to use */
#define HRTIME_CLOCK_ID_STR "CLOCK_UPTIME_PRECISE"

/* define if h_errno is declared (in some way) in a system header file */
#define H_ERRNO_DECLARED 1

/* Define to monotonic clock id to use */
#define MONOTONIC_CLOCK_ID CLOCK_UPTIME

/* Define as a string of monotonic clock id to use */
#define MONOTONIC_CLOCK_ID_STR "CLOCK_UPTIME"

/* Define if netdb.h needs struct sockaddr_in ans in.h CAN be included before
   */
/* #undef NETDB_H_NEEDS_IN_H */

/* Define if floating points exceptions are non-existing/not reliable */
#define NO_FPE_SIGNALS 

/* Defined if no found C compiler can handle jump tables */
/* #undef NO_JUMP_TABLE */

/* Define if you dont have salen */
/* #undef NO_SA_LEN */

/* Define if you want to implement erts_os_monotonic_time() using
   clock_gettime() */
#define OS_MONOTONIC_TIME_USING_CLOCK_GETTIME 1

/* Define if you want to implement erts_os_monotonic_time() using gethrtime()
   */
/* #undef OS_MONOTONIC_TIME_USING_GETHRTIME */

/* Define if you want to implement erts_os_monotonic_time() using mach
   clock_get_time() */
/* #undef OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME */

/* Define if you want to implement erts_os_monotonic_time() using times() */
/* #undef OS_MONOTONIC_TIME_USING_TIMES */

/* Define if you want to implement erts_os_system_time() using gettimeofday()
   */
/* #undef OS_SYSTEM_TIME_GETTIMEOFDAY */

/* Define if you want to implement erts_os_system_time() using clock_gettime()
   */
#define OS_SYSTEM_TIME_USING_CLOCK_GETTIME 1

/* Define if you want to implement erts_os_system_time() using mach
   clock_get_time() */
/* #undef OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME */

/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT ""

/* Define to the full name of this package. */
#define PACKAGE_NAME ""

/* Define to the full name and version of this package. */
#define PACKAGE_STRING ""

/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME ""

/* Define to the version of this package. */
#define PACKAGE_VERSION ""

/* Define if you wish to redefine FD_SETSIZE to be able to select on more fd
   */
/* #undef REDEFINE_FD_SETSIZE */

/* Define as the return type of signal handlers (`int' or `void'). */
#define RETSIGTYPE void

/* Define the sbrk() argument type. */
#define SBRK_ARG_TYPE intptr_t

/* Define the sbrk() return type. */
#define SBRK_RET_TYPE void *

/* The size of a `AO_t', as computed by sizeof. */
/* #undef SIZEOF_AO_T */

/* The size of a `char', as computed by sizeof. */
#define SIZEOF_CHAR 1

/* The size of a `int', as computed by sizeof. */
#define SIZEOF_INT 4

/* The size of a `long', as computed by sizeof. */
#define SIZEOF_LONG 4

/* The size of a `long long', as computed by sizeof. */
#define SIZEOF_LONG_LONG 8

/* The size of a `off_t', as computed by sizeof. */
#define SIZEOF_OFF_T 8

/* The size of a `short', as computed by sizeof. */
#define SIZEOF_SHORT 2

/* The size of a `size_t', as computed by sizeof. */
#define SIZEOF_SIZE_T 4

/* The size of a `time_t', as computed by sizeof. */
#define SIZEOF_TIME_T 8

/* The size of a `void *', as computed by sizeof. */
#define SIZEOF_VOID_P 4

/* The size of a `__int128_t', as computed by sizeof. */
#define SIZEOF___INT128_T 0

/* The size of a `__int64', as computed by sizeof. */
#define SIZEOF___INT64 0

/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1

/* define if the variable sys_errlist is declared in a system header file */
#define SYS_ERRLIST_DECLARED 

/* Define if you want to implement erts_os_hrtime() using clock_gettime() */
/* #undef SYS_HRTIME_USING_CLOCK_GETTIME */

/* Define if you want to implement erts_os_hrtime() using gethrtime() */
/* #undef SYS_HRTIME_USING_GETHRTIME */

/* Define if you want to implement erts_os_hrtime() using mach
   clock_get_time() */
/* #undef SYS_HRTIME_USING_MACH_CLOCK_GET_TIME */

/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#define TIME_WITH_SYS_TIME 1

/* Define to 1 if your <sys/time.h> declares `struct tm'. */
/* #undef TM_IN_SYS_TIME */

/* Define if you want to use dtrace for dynamic tracing */
/* #undef USE_DTRACE */

/* Define if you want to use dynamic tracing */
/* #undef USE_DYNAMIC_TRACE */

/* Define if you have matherr() function and struct exception type */
/* #undef USE_MATHERR */

/* Define if select() should be used instead of poll() */
/* #undef USE_SELECT */

/* Define if you want to use systemtap for dynamic tracing */
/* #undef USE_SYSTEMTAP */

/* Define to enable VM dynamic trace probes */
/* #undef USE_VM_PROBES */

/* Define to wall clock id to use */
#define WALL_CLOCK_ID CLOCK_REALTIME

/* Define as a string of wall clock id to use */
#define WALL_CLOCK_ID_STR "CLOCK_REALTIME"

/* Define if windows.h includes winsock2.h */
/* #undef WINDOWS_H_INCLUDES_WINSOCK2_H */

/* Define to 1 if your processor stores words with the most significant byte
   first (like Motorola and SPARC, unlike Intel and VAX). */
/* #undef WORDS_BIGENDIAN */

/* Define to empty if `const' does not conform to ANSI C. */
/* #undef const */

/* Define to `long' if <sys/types.h> does not define. */
/* #undef off_t */

/* Define to `int' if <sys/types.h> does not define. */
/* #undef pid_t */

/* Define to `unsigned' if <sys/types.h> does not define. */
/* #undef size_t */

/* Define as `fork' if `vfork' does not work. */
/* #undef vfork */


/* Redefine in6_addr. XXX this should be moved to the files where it's used? */
#ifdef HAVE_IN_ADDR6_STRUCT
#define in6_addr in_addr6
#endif
# 1245 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

/* Define a reasonable default for INADDR_LOOPBACK */
/* XXX this should be moved to the files where it's used? */
#ifdef HAVE_NO_INADDR_LOOPBACK
#define INADDR_LOOPBACK (u_long)0x7F000001
#endif
# 1251 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#ifdef REDEFINE_FD_SETSIZE
#define FD_SETSIZE 1024
#endif
# 1255 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
#define HAVE_GETHRVTIME
#endif
# 1259 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#if !defined(HAVE_ISFINITE) && !defined(HAVE_FINITE)
# if defined(HAVE_ISINF) && defined(HAVE_ISNAN)
#  define USE_ISINF_ISNAN
# endif
# 1264 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"
#endif
# 1265 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#if defined(DEBUG) && defined(USE_THREADS) && !defined(ERTS_ENABLE_LOCK_CHECK)
#define ERTS_ENABLE_LOCK_CHECK 1
#endif
# 1269 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

# 30 "beam/erl_driver.h" 2
#endif
# 31 "beam/erl_driver.h"

#define ERL_DRV_DEPRECATED_FUNC
#ifdef __GNUC__
#  if __GNUC__ >= 3
#    undef ERL_DRV_DEPRECATED_FUNC
#    define ERL_DRV_DEPRECATED_FUNC __attribute__((deprecated))
#  endif
# 38 "beam/erl_driver.h"
#endif
# 39 "beam/erl_driver.h"

/* This is OK to override by the NIF/driver implementor */
#if defined(HALFWORD_HEAP_EMULATOR_SAVED__) && !defined(HALFWORD_HEAP_EMULATOR)
#define HALFWORD_HEAP_EMULATOR HALFWORD_HEAP_EMULATOR_SAVED__
#endif
# 44 "beam/erl_driver.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_drv_nif.h"
#endif /* expanded by -frewrite-includes */
# 45 "beam/erl_driver.h"
# 1 "beam/erl_drv_nif.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2010. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Common structures for both erl_driver.h and erl_nif.h
 */

#ifndef __ERL_DRV_NIF_H__
#define __ERL_DRV_NIF_H__

typedef struct {
    int driver_major_version;
    int driver_minor_version;
    char *erts_version;
    char *otp_release;
    int thread_support;
    int smp_support;
    int async_threads;
    int scheduler_threads;
    int nif_major_version;
    int nif_minor_version;
    int dirty_scheduler_support;
}  ErlDrvSysInfo;

typedef struct {
    int suggested_stack_size;
} ErlDrvThreadOpts;

#if defined(ERL_DRV_DIRTY_SCHEDULER_SUPPORT) || defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT)
typedef enum {
    ERL_DRV_DIRTY_JOB_CPU_BOUND = 1,
    ERL_DRV_DIRTY_JOB_IO_BOUND  = 2
} ErlDrvDirtyJobFlags;
#endif
# 52 "beam/erl_drv_nif.h"

#ifdef SIZEOF_CHAR
#  define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR
#  undef SIZEOF_CHAR
#endif
# 57 "beam/erl_drv_nif.h"
#ifdef SIZEOF_SHORT
#  define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT
#  undef SIZEOF_SHORT
#endif
# 61 "beam/erl_drv_nif.h"
#ifdef SIZEOF_INT
#  define SIZEOF_INT_SAVED__ SIZEOF_INT
#  undef SIZEOF_INT
#endif
# 65 "beam/erl_drv_nif.h"
#ifdef SIZEOF_LONG
#  define SIZEOF_LONG_SAVED__ SIZEOF_LONG
#  undef SIZEOF_LONG
#endif
# 69 "beam/erl_drv_nif.h"
#ifdef SIZEOF_LONG_LONG
#  define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG
#  undef SIZEOF_LONG_LONG
#endif
# 73 "beam/erl_drv_nif.h"
#ifdef HALFWORD_HEAP_EMULATOR
#  define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR
#  undef HALFWORD_HEAP_EMULATOR
#endif
# 77 "beam/erl_drv_nif.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_int_sizes_config.h"
#endif /* expanded by -frewrite-includes */
# 77 "beam/erl_drv_nif.h"
# 1 "../include/armv6-portbld-freebsd11.0/erl_int_sizes_config.h" 1
/* include/armv6-portbld-freebsd11.0/erl_int_sizes_config.h.  Generated by configure.  */
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2004-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* The number of bytes in a char.  */
#define SIZEOF_CHAR 1

/* The number of bytes in a short.  */
#define SIZEOF_SHORT 2

/* The number of bytes in a int.  */
#define SIZEOF_INT 4

/* The number of bytes in a long.  */
#define SIZEOF_LONG 4

/* The number of bytes in a long long.  */
#define SIZEOF_LONG_LONG 8

/* The size of a pointer. */
#define SIZEOF_VOID_P 4

/* Define if building a halfword-heap 64bit emulator (needed for NIF's) */
/* #undef HALFWORD_HEAP_EMULATOR */
# 78 "beam/erl_drv_nif.h" 2
#if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR
#  error SIZEOF_CHAR mismatch
#endif
# 81 "beam/erl_drv_nif.h"
#if defined(SIZEOF_SHORT_SAVED__) && SIZEOF_SHORT_SAVED__ != SIZEOF_SHORT
#  error SIZEOF_SHORT mismatch
#endif
# 84 "beam/erl_drv_nif.h"
#if defined(SIZEOF_INT_SAVED__) && SIZEOF_INT_SAVED__ != SIZEOF_INT
#  error SIZEOF_INT mismatch
#endif
# 87 "beam/erl_drv_nif.h"
#if defined(SIZEOF_LONG_SAVED__) && SIZEOF_LONG_SAVED__ != SIZEOF_LONG
#  error SIZEOF_LONG mismatch
#endif
# 90 "beam/erl_drv_nif.h"
#if defined(SIZEOF_LONG_LONG_SAVED__) && SIZEOF_LONG_LONG_SAVED__ != SIZEOF_LONG_LONG
#  error SIZEOF_LONG_LONG mismatch
#endif
# 93 "beam/erl_drv_nif.h"

#if !defined(__GNUC__) && (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_))
typedef unsigned __int64 ErlNapiUInt64;
typedef signed __int64 ErlNapiSInt64;
#define ERL_NAPI_SINT64_MAX__ 9223372036854775807i64
#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1i64)
#elif SIZEOF_LONG == 8
# 100 "beam/erl_drv_nif.h"
typedef unsigned long ErlNapiUInt64;
typedef signed long ErlNapiSInt64;
#define ERL_NAPI_SINT64_MAX__ 9223372036854775807L
#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1L)
#elif SIZEOF_LONG_LONG == 8
# 105 "beam/erl_drv_nif.h"
typedef unsigned long long ErlNapiUInt64;
typedef signed long long ErlNapiSInt64;
#define ERL_NAPI_SINT64_MAX__ 9223372036854775807LL
#define ERL_NAPI_SINT64_MIN__ (-ERL_NAPI_SINT64_MAX__ - 1LL)
#else
# 110 "beam/erl_drv_nif.h"
#  error No 64-bit integer type
#endif
# 112 "beam/erl_drv_nif.h"

#if SIZEOF_VOID_P == 8
typedef ErlNapiUInt64 ErlNapiUInt;
typedef ErlNapiSInt64 ErlNapiSInt;
#elif SIZEOF_VOID_P == 4
# 117 "beam/erl_drv_nif.h"
#  if SIZEOF_LONG == SIZEOF_VOID_P
typedef unsigned long ErlNapiUInt;
typedef signed long ErlNapiSInt;
#  elif SIZEOF_INT == SIZEOF_VOID_P
# 121 "beam/erl_drv_nif.h"
typedef unsigned int ErlNapiUInt;
typedef signed int ErlNapiSInt;
#  else
# 124 "beam/erl_drv_nif.h"
#    error No 32-bit integer type
#  endif
# 126 "beam/erl_drv_nif.h"
#else
# 127 "beam/erl_drv_nif.h"
#  error Not support arch
#endif
# 129 "beam/erl_drv_nif.h"

#define ERTS_NAPI_TIME_ERROR__ ERL_NAPI_SINT64_MIN__

#define ERTS_NAPI_SEC__		0
#define ERTS_NAPI_MSEC__	1
#define ERTS_NAPI_USEC__	2
#define ERTS_NAPI_NSEC__	3

#endif  /* __ERL_DRV_NIF_H__ */
# 138 "beam/erl_drv_nif.h"




# 46 "beam/erl_driver.h" 2

#if 0 /* expanded by -frewrite-includes */
#include <stdlib.h>
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_driver.h"
# 48 "beam/erl_driver.h"
#if 0 /* expanded by -frewrite-includes */
#include <sys/types.h>	/* ssize_t */
#endif /* expanded by -frewrite-includes */
# 48 "beam/erl_driver.h"
# 49 "beam/erl_driver.h"

#if defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)
#ifndef STATIC_ERLANG_DRIVER
   /* Windows dynamic drivers, everything is different... */
#define ERL_DRIVER_TYPES_ONLY
#define WIN32_DYNAMIC_ERL_DRIVER
#endif
# 56 "beam/erl_driver.h"
/*
 * This structure can be cast to a WSABUF structure.
 */
typedef struct _SysIOVec {
    unsigned long iov_len;
    char* iov_base;
} SysIOVec;
#else  /* Unix */
# 64 "beam/erl_driver.h"
#  ifdef HAVE_SYS_UIO_H
#if 0 /* expanded by -frewrite-includes */
#    include <sys/types.h>
#endif /* expanded by -frewrite-includes */
# 65 "beam/erl_driver.h"
# 66 "beam/erl_driver.h"
#if 0 /* expanded by -frewrite-includes */
#    include <sys/uio.h>
#endif /* expanded by -frewrite-includes */
# 66 "beam/erl_driver.h"
# 1 "/usr/include/sys/uio.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1993, 1994
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)uio.h	8.5 (Berkeley) 2/22/94
 * $FreeBSD: head/sys/sys/uio.h 291716 2015-12-03 20:54:55Z ken $
 */

#ifndef _SYS_UIO_H_
#define	_SYS_UIO_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/cdefs.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/uio.h" 3 4
# 37 "/usr/include/sys/uio.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 37 "/usr/include/sys/uio.h" 3 4
# 38 "/usr/include/sys/uio.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#include <sys/_iovec.h>
#endif /* expanded by -frewrite-includes */
# 38 "/usr/include/sys/uio.h" 3 4
# 1 "/usr/include/sys/_iovec.h" 1 3 4
/*-
 * Copyright (c) 1982, 1986, 1993, 1994
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)uio.h	8.5 (Berkeley) 2/22/94
 * $FreeBSD: head/sys/sys/_iovec.h 139825 2005-01-07 02:29:27Z imp $
 */

#ifndef _SYS__IOVEC_H_
#define	_SYS__IOVEC_H_

#if 0 /* expanded by -frewrite-includes */
#include <sys/_types.h>
#endif /* expanded by -frewrite-includes */
# 36 "/usr/include/sys/_iovec.h" 3 4
# 37 "/usr/include/sys/_iovec.h" 3 4

#ifndef _SIZE_T_DECLARED
typedef	__size_t	size_t;
#define	_SIZE_T_DECLARED
#endif
# 42 "/usr/include/sys/_iovec.h" 3 4

struct iovec {
	void	*iov_base;	/* Base address. */
	size_t	 iov_len;	/* Length. */
};

#endif /* !_SYS__IOVEC_H_ */
# 49 "/usr/include/sys/_iovec.h" 3 4
# 39 "/usr/include/sys/uio.h" 2 3 4

#ifndef _SSIZE_T_DECLARED
typedef	__ssize_t	ssize_t;
#define	_SSIZE_T_DECLARED
#endif
# 44 "/usr/include/sys/uio.h" 3 4

#ifndef _OFF_T_DECLARED
typedef	__off_t	off_t;
#define	_OFF_T_DECLARED
#endif
# 49 "/usr/include/sys/uio.h" 3 4

#if __BSD_VISIBLE
enum	uio_rw { UIO_READ, UIO_WRITE };

/* Segment flag values. */
enum uio_seg {
	UIO_USERSPACE,		/* from user data space */
	UIO_SYSSPACE,		/* from system space */
	UIO_NOCOPY		/* don't copy, already in object */
};
#endif
# 60 "/usr/include/sys/uio.h" 3 4

#ifdef _KERNEL

struct uio {
	struct	iovec *uio_iov;		/* scatter/gather list */
	int	uio_iovcnt;		/* length of scatter/gather list */
	off_t	uio_offset;		/* offset in target object */
	ssize_t	uio_resid;		/* remaining bytes to process */
	enum	uio_seg uio_segflg;	/* address space */
	enum	uio_rw uio_rw;		/* operation */
	struct	thread *uio_td;		/* owner */
};

/*
 * Limits
 *
 * N.B.: UIO_MAXIOV must be no less than IOV_MAX from <sys/syslimits.h>
 * which in turn must be no less than _XOPEN_IOV_MAX from <limits.h>.  If
 * we ever make this tunable (probably pointless), then IOV_MAX should be
 * removed from <sys/syslimits.h> and applications would be expected to use
 * sysconf(3) to find out the correct value, or else assume the worst
 * (_XOPEN_IOV_MAX).  Perhaps UIO_MAXIOV should be simply defined as
 * IOV_MAX.
 */
#define UIO_MAXIOV	1024		/* max 1K of iov's */

struct vm_object;
struct vm_page;
struct bus_dma_segment;

struct uio *cloneuio(struct uio *uiop);
int	copyinfrom(const void * __restrict src, void * __restrict dst,
	    size_t len, int seg);
int	copyiniov(const struct iovec *iovp, u_int iovcnt, struct iovec **iov,
	    int error);
int	copyinstrfrom(const void * __restrict src, void * __restrict dst,
	    size_t len, size_t * __restrict copied, int seg);
int	copyinuio(const struct iovec *iovp, u_int iovcnt, struct uio **uiop);
int	copyout_map(struct thread *td, vm_offset_t *addr, size_t sz);
int	copyout_unmap(struct thread *td, vm_offset_t addr, size_t sz);
int	physcopyin(void *src, vm_paddr_t dst, size_t len);
int	physcopyout(vm_paddr_t src, void *dst, size_t len);
int	physcopyin_vlist(struct bus_dma_segment *src, off_t offset,
	    vm_paddr_t dst, size_t len);
int	physcopyout_vlist(vm_paddr_t src, struct bus_dma_segment *dst,
	    off_t offset, size_t len);
int	uiomove(void *cp, int n, struct uio *uio);
int	uiomove_frombuf(void *buf, int buflen, struct uio *uio);
int	uiomove_fromphys(struct vm_page *ma[], vm_offset_t offset, int n,
	    struct uio *uio);
int	uiomove_nofault(void *cp, int n, struct uio *uio);
int	uiomove_object(struct vm_object *obj, off_t obj_size, struct uio *uio);

#else /* !_KERNEL */
# 114 "/usr/include/sys/uio.h" 3 4

__BEGIN_DECLS
ssize_t	readv(int, const struct iovec *, int);
ssize_t	writev(int, const struct iovec *, int);
#if __BSD_VISIBLE
ssize_t	preadv(int, const struct iovec *, int, off_t);
ssize_t	pwritev(int, const struct iovec *, int, off_t);
#endif
# 122 "/usr/include/sys/uio.h" 3 4
__END_DECLS

#endif /* _KERNEL */
# 125 "/usr/include/sys/uio.h" 3 4

#endif /* !_SYS_UIO_H_ */
# 127 "/usr/include/sys/uio.h" 3 4
# 67 "beam/erl_driver.h" 2
typedef struct iovec SysIOVec;
#  else
# 69 "beam/erl_driver.h"
typedef struct {
    char* iov_base;
    size_t iov_len;
} SysIOVec;
#  endif
# 74 "beam/erl_driver.h"
#endif
# 75 "beam/erl_driver.h"

#ifndef EXTERN
#  ifdef __cplusplus
#    define EXTERN extern "C"
#  else
# 80 "beam/erl_driver.h"
#    define EXTERN extern
#  endif
# 82 "beam/erl_driver.h"
#endif
# 83 "beam/erl_driver.h"

/* Values for mode arg to driver_select() */
#define ERL_DRV_READ  (1 << 0)
#define ERL_DRV_WRITE (1 << 1)
#define ERL_DRV_USE   (1 << 2)
#define ERL_DRV_USE_NO_CALLBACK (ERL_DRV_USE | (1 << 3))

/* Old deprecated */
#define DO_READ  ERL_DRV_READ
#define DO_WRITE ERL_DRV_WRITE

#define ERL_DRV_EXTENDED_MARKER		(0xfeeeeeed)
#define ERL_DRV_EXTENDED_MAJOR_VERSION	3
#define ERL_DRV_EXTENDED_MINOR_VERSION	3

/*
 * The emulator will refuse to load a driver with a major version
 * lower than ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD. The load
 * may however fail if user have not removed use of deprecated
 * symbols.
 *
 * The ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD have to allow
 * loading of drivers built at least two major OTP releases
 * ago.
 *
 * Bump of major version to 3 happened in OTP 17. That is, in
 * OTP 19 we can increase ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD
 * to 3.
 */
#define ERL_DRV_MIN_REQUIRED_MAJOR_VERSION_ON_LOAD 2

/*
 * The emulator will refuse to load a driver with different major
 * version than the one used by the emulator.
 */


/* Values for set_port_control_flags() */

#define PORT_CONTROL_FLAG_BINARY	(1 << 0)
#define PORT_CONTROL_FLAG_HEAVY		(1 << 1)

/* Values for get_port_flags() */

#define PORT_FLAG_BINARY                (1 << 0)
#define PORT_FLAG_LINE                  (1 << 1)


#define ERL_DRV_FLAG_USE_PORT_LOCKING	(1 << 0)
#define ERL_DRV_FLAG_SOFT_BUSY		(1 << 1)
#define ERL_DRV_FLAG_NO_BUSY_MSGQ	(1 << 2)

/*
 * Integer types
 */

typedef ErlNapiUInt64 ErlDrvUInt64;
typedef ErlNapiSInt64 ErlDrvSInt64;
typedef ErlNapiUInt ErlDrvUInt;
typedef ErlNapiSInt ErlDrvSInt;
typedef ErlNapiUInt ErlDrvTermData;

#if defined(__WIN32__) || defined(_WIN32)
typedef ErlDrvUInt ErlDrvSizeT;
typedef ErlDrvSInt ErlDrvSSizeT;
#else
# 149 "beam/erl_driver.h"
typedef size_t ErlDrvSizeT;
typedef ssize_t ErlDrvSSizeT;
#endif
# 152 "beam/erl_driver.h"

/*
 * A binary as seen in a driver. Note that a binary should never be
 * altered by the driver when it has been sent to Erlang.
 */

typedef struct erl_drv_binary {
    ErlDrvSInt orig_size;        /* total length of binary */
    char orig_bytes[1];   /* the data (char instead of byte!) */
} ErlDrvBinary;


/*
 * Note: These types are incomplete to catch type errors easier.
 */

typedef struct _erl_drv_data* ErlDrvData; /* Data to be used by the driver itself. */
#ifndef ERL_SYS_DRV
typedef struct _erl_drv_event* ErlDrvEvent; /* An event to be selected on. */
#endif
# 172 "beam/erl_driver.h"
typedef struct _erl_drv_port* ErlDrvPort; /* A port descriptor. */
typedef struct _erl_drv_port* ErlDrvThreadData; /* Thread data. */

#if !defined(__WIN32__) && !defined(_WIN32) && !defined(_WIN32_) && !defined(USE_SELECT)
struct erl_drv_event_data {
    short events;
    short revents;
};
#endif
# 181 "beam/erl_driver.h"
typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */

/*
 * A driver monitor
 */
typedef struct {
    unsigned char data[sizeof(void *)*4];
} ErlDrvMonitor;

typedef struct {
    unsigned long megasecs;
    unsigned long secs;
    unsigned long microsecs;
} ErlDrvNowData;

typedef ErlDrvSInt64 ErlDrvTime;

#define ERL_DRV_TIME_ERROR ((ErlDrvSInt64) ERTS_NAPI_TIME_ERROR__)

typedef enum {
    ERL_DRV_SEC = ERTS_NAPI_SEC__,
    ERL_DRV_MSEC = ERTS_NAPI_MSEC__,
    ERL_DRV_USEC = ERTS_NAPI_USEC__,
    ERL_DRV_NSEC = ERTS_NAPI_NSEC__
} ErlDrvTimeUnit;

/*
 * Error codes that can be return from driver.
 */

/*
 * Exception code from open_port/2 will be {'EXIT',{einval,Where}}.
 */
#define ERL_DRV_ERROR_GENERAL ((ErlDrvData) -1)

/*
 * Exception code from open_port/2 will be {'EXIT',{Errno,Where}},
 * where Errno is a textual representation of the errno variable
 * (e.g. eacces if errno is EACCES).
 */
#define ERL_DRV_ERROR_ERRNO ((ErlDrvData) -2)

/*
 * Exception code from open_port/2 will be {'EXIT',{badarg,Where}}.
 */
#define ERL_DRV_ERROR_BADARG ((ErlDrvData) -3)

typedef struct erl_io_vec {
    int vsize;			/* length of vectors */
    ErlDrvSizeT size;		/* total size in bytes */
    SysIOVec* iov;
    ErlDrvBinary** binv;
} ErlIOVec;

/*
 * erl driver thread types
 */

typedef struct ErlDrvTid_ *ErlDrvTid;
typedef struct ErlDrvMutex_ ErlDrvMutex;
typedef struct ErlDrvCond_ ErlDrvCond;
typedef struct ErlDrvRWLock_ ErlDrvRWLock;
typedef int ErlDrvTSDKey;

/*
 * 
 */
typedef struct erl_drv_port_data_lock * ErlDrvPDL;

/*
 * This structure defines a driver.
 */

typedef struct erl_drv_entry {
    int (*init)(void);		/* called at system start up for statically
				   linked drivers, and after loading for
				   dynamically loaded drivers */ 

#ifndef ERL_SYS_DRV
    ErlDrvData (*start)(ErlDrvPort port, char *command);
				/* called when open_port/2 is invoked.
				   return value -1 means failure. */
#else
# 264 "beam/erl_driver.h"
    ErlDrvData (*start)(ErlDrvPort port, char *command, SysDriverOpts* opts);
				/* special options, only for system driver */
#endif
# 267 "beam/erl_driver.h"
    void (*stop)(ErlDrvData drv_data);
                                /* called when port is closed, and when the
				   emulator is halted. */
    void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
				/* called when we have output from erlang to
				   the port */
    void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
				/* called when we have input from one of 
				   the driver's handles */
    void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
				/* called when output is possible to one of 
				   the driver's handles */
    char *driver_name;		/* name supplied as command 
				   in open_port XXX ? */
    void (*finish)(void);        /* called before unloading the driver -
				   DYNAMIC DRIVERS ONLY */
    void *handle;		/* Reserved -- Used by emulator internally */
    ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command,
			    char *buf, ErlDrvSizeT len, char **rbuf,
			    ErlDrvSizeT rlen); /* "ioctl" for drivers - invoked by
						  port_control/3 */
    void (*timeout)(ErlDrvData drv_data);	/* Handling of timeout in driver */
    void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev);
				/* called when we have output from erlang
				   to the port */
    void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data);
    void (*flush)(ErlDrvData drv_data);
                                /* called when the port is about to be 
				   closed, and there is data in the 
				   driver queue that needs to be flushed
				   before 'stop' can be called */
    ErlDrvSSizeT (*call)(ErlDrvData drv_data,
			 unsigned int command, char *buf, ErlDrvSizeT len,
			 char **rbuf, ErlDrvSizeT rlen,
			 unsigned int *flags); /* Works mostly like 'control',
						  a synchronous
						  call into the driver. */
    void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
		  ErlDrvEventData event_data);
                                /* Called when an event selected by 
				   driver_event() has occurred */
    int extended_marker;	/* ERL_DRV_EXTENDED_MARKER */
    int major_version;		/* ERL_DRV_EXTENDED_MAJOR_VERSION */
    int minor_version;		/* ERL_DRV_EXTENDED_MINOR_VERSION */
    int driver_flags;		/* ERL_DRV_FLAGs */
    void *handle2;              /* Reserved -- Used by emulator internally */
    void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
                                /* Called when a process monitor fires */
    void (*stop_select)(ErlDrvEvent event, void* reserved);
    	                        /* Called on behalf of driver_select when
				   it is safe to release 'event'. A typical
				   unix driver would call close(event) */
    void (*emergency_close)(ErlDrvData drv_data);
                                /* called when the port is closed abruptly.
				   specifically when erl_crash_dump is called. */
    /* When adding entries here, dont forget to pad in obsolete/driver.h */
} ErlDrvEntry;

/*
 * This macro is used to name a dynamic driver's init function in
 * a way that doesn't lead to conflicts. This is crucial when using
 * operating systems that has one namespace for all symbols
 * (e.g. VxWorks). Example: if you have an dynamic driver C source
 * file named echo_drv.c, you use the macro like this:
 * 
 *    DRIVER_INIT(echo_drv)
 *    {
 *	 ....
 *    }
 *
 * This function will be called by the Erlang I/O system when the driver is loaded.
 * It must initialize a ErlDrvEntry structure and return a pointer to it.
 */

#ifdef STATIC_ERLANG_DRIVER
#  define ERLANG_DRIVER_NAME(NAME) NAME ## _driver_init
#else
# 344 "beam/erl_driver.h"
#  define ERLANG_DRIVER_NAME(NAME) driver_init
#endif
# 346 "beam/erl_driver.h"

/* For windows dynamic drivers */
#ifndef ERL_DRIVER_TYPES_ONLY

#if defined(__WIN32__)
#  define DRIVER_INIT(DRIVER_NAME) \
  __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void);	\
    __declspec(dllexport) ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
#else 
# 355 "beam/erl_driver.h"
#  define DRIVER_INIT(DRIVER_NAME) \
    ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void); \
    ErlDrvEntry* ERLANG_DRIVER_NAME(DRIVER_NAME)(void)
#endif
# 359 "beam/erl_driver.h"

#define ERL_DRV_BUSY_MSGQ_DISABLED	(~((ErlDrvSizeT) 0))
#define ERL_DRV_BUSY_MSGQ_READ_ONLY	((ErlDrvSizeT) 0)
#define ERL_DRV_BUSY_MSGQ_LIM_MAX	(ERL_DRV_BUSY_MSGQ_DISABLED - 1)
#define ERL_DRV_BUSY_MSGQ_LIM_MIN	((ErlDrvSizeT) 1)

/*
 * These are the functions available for driver writers.
 */
EXTERN void erl_drv_busy_msgq_limits(ErlDrvPort port,
				     ErlDrvSizeT *low,
				     ErlDrvSizeT *high);

EXTERN int driver_select(ErlDrvPort port, ErlDrvEvent event, int mode, int on);
EXTERN int driver_event(ErlDrvPort port, ErlDrvEvent event, 
			ErlDrvEventData event_data);

EXTERN int driver_output(ErlDrvPort port, char *buf, ErlDrvSizeT len);
EXTERN int driver_output2(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen,
			  char *buf, ErlDrvSizeT len);
EXTERN int driver_output_binary(ErlDrvPort port, char *hbuf, ErlDrvSizeT hlen,
				ErlDrvBinary* bin,
				ErlDrvSizeT offset, ErlDrvSizeT len);
EXTERN int driver_outputv(ErlDrvPort port, char* hbuf, ErlDrvSizeT hlen,
			  ErlIOVec *ev, ErlDrvSizeT skip);
EXTERN ErlDrvSizeT driver_vec_to_buf(ErlIOVec *ev, char *buf, ErlDrvSizeT len);
EXTERN int driver_set_timer(ErlDrvPort port, unsigned long time);
EXTERN int driver_cancel_timer(ErlDrvPort port);
EXTERN int driver_read_timer(ErlDrvPort port, unsigned long *time_left);

/*
 * Inform runtime system about lengthy work.
 */
EXTERN int erl_drv_consume_timeslice(ErlDrvPort port, int percent);

/*
 * Get plain-text error message from within a driver
 */
EXTERN char* erl_errno_id(int error);

/*
 * The following functions are used to initiate a close of a port
 * from a driver.
 */
EXTERN int driver_failure_eof(ErlDrvPort port);
EXTERN int driver_failure_atom(ErlDrvPort port, char *string);
EXTERN int driver_failure_posix(ErlDrvPort port, int error);
EXTERN int driver_failure(ErlDrvPort port, int error);
EXTERN int driver_exit (ErlDrvPort port, int err);


/*
 * Port Data Lock
 */

EXTERN ErlDrvPDL driver_pdl_create(ErlDrvPort);
EXTERN void driver_pdl_lock(ErlDrvPDL);
EXTERN void driver_pdl_unlock(ErlDrvPDL);
EXTERN ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL);
EXTERN ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL);
EXTERN ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL);

/*
 * Process monitors
 */
EXTERN int 
driver_monitor_process(ErlDrvPort port, ErlDrvTermData process, 
		       ErlDrvMonitor *monitor);
EXTERN int 
driver_demonitor_process(ErlDrvPort port, const ErlDrvMonitor *monitor);
EXTERN ErlDrvTermData 
driver_get_monitored_process(ErlDrvPort port, const ErlDrvMonitor *monitor);
EXTERN int driver_compare_monitors(const ErlDrvMonitor *monitor1,
				   const ErlDrvMonitor *monitor2);

/*
 * Port attributes
 */
EXTERN void set_busy_port(ErlDrvPort port, int on);
EXTERN void set_port_control_flags(ErlDrvPort port, int flags);

EXTERN int  get_port_flags(ErlDrvPort port);


/* Binary interface */

/*
 * NOTE: DO NOT overwrite a binary with new data (if the data is delivered);
 * since the binary is a shared object it MUST be written once.
 */

EXTERN ErlDrvBinary* driver_alloc_binary(ErlDrvSizeT size);
EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, ErlDrvSizeT size);
EXTERN void driver_free_binary(ErlDrvBinary *bin);

/* Referenc count on driver binaries */
EXTERN ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp);
EXTERN ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp);
EXTERN ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp);

/* Allocation interface */
EXTERN void *driver_alloc(ErlDrvSizeT size);
EXTERN void *driver_realloc(void *ptr, ErlDrvSizeT size);
EXTERN void driver_free(void *ptr);

/* Queue interface */
EXTERN int driver_enq(ErlDrvPort port, char* buf, ErlDrvSizeT len);
EXTERN int driver_pushq(ErlDrvPort port, char* buf, ErlDrvSizeT len);
EXTERN ErlDrvSizeT driver_deq(ErlDrvPort port, ErlDrvSizeT size);
EXTERN ErlDrvSizeT driver_sizeq(ErlDrvPort port);
EXTERN int driver_enq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset,
			  ErlDrvSizeT len);
EXTERN int driver_pushq_bin(ErlDrvPort port, ErlDrvBinary *bin, ErlDrvSizeT offset,
			    ErlDrvSizeT len);

EXTERN ErlDrvSizeT driver_peekqv(ErlDrvPort port, ErlIOVec *ev);
EXTERN SysIOVec* driver_peekq(ErlDrvPort port, int *vlen);
EXTERN int driver_enqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip);
EXTERN int driver_pushqv(ErlDrvPort port, ErlIOVec *ev, ErlDrvSizeT skip);

/*
 * Add and remove driver entries.
 */
EXTERN void add_driver_entry(ErlDrvEntry *de);
EXTERN int remove_driver_entry(ErlDrvEntry *de);

/*
 * System info
 */
EXTERN void driver_system_info(ErlDrvSysInfo *sip, size_t si_size);

/*
 * erl driver thread functions.
 */

EXTERN ErlDrvMutex *erl_drv_mutex_create(char *name);
EXTERN void erl_drv_mutex_destroy(ErlDrvMutex *mtx);
EXTERN int erl_drv_mutex_trylock(ErlDrvMutex *mtx);
EXTERN void erl_drv_mutex_lock(ErlDrvMutex *mtx);
EXTERN void erl_drv_mutex_unlock(ErlDrvMutex *mtx);
EXTERN ErlDrvCond *erl_drv_cond_create(char *name);
EXTERN void erl_drv_cond_destroy(ErlDrvCond *cnd);
EXTERN void erl_drv_cond_signal(ErlDrvCond *cnd);
EXTERN void erl_drv_cond_broadcast(ErlDrvCond *cnd);
EXTERN void erl_drv_cond_wait(ErlDrvCond *cnd, ErlDrvMutex *mtx);
EXTERN ErlDrvRWLock *erl_drv_rwlock_create(char *name);
EXTERN void erl_drv_rwlock_destroy(ErlDrvRWLock *rwlck);
EXTERN int erl_drv_rwlock_tryrlock(ErlDrvRWLock *rwlck);
EXTERN void erl_drv_rwlock_rlock(ErlDrvRWLock *rwlck);
EXTERN void erl_drv_rwlock_runlock(ErlDrvRWLock *rwlck);
EXTERN int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *rwlck);
EXTERN void erl_drv_rwlock_rwlock(ErlDrvRWLock *rwlck);
EXTERN void erl_drv_rwlock_rwunlock(ErlDrvRWLock *rwlck);
EXTERN int erl_drv_tsd_key_create(char *name, ErlDrvTSDKey *key);
EXTERN void erl_drv_tsd_key_destroy(ErlDrvTSDKey key);
EXTERN void erl_drv_tsd_set(ErlDrvTSDKey key, void *data);
EXTERN void *erl_drv_tsd_get(ErlDrvTSDKey key);
EXTERN ErlDrvThreadOpts *erl_drv_thread_opts_create(char *name);
EXTERN void erl_drv_thread_opts_destroy(ErlDrvThreadOpts *opts);
EXTERN int erl_drv_thread_create(char *name,
				 ErlDrvTid *tid,
				 void * (*func)(void *),
				 void *args,
				 ErlDrvThreadOpts *opts);
EXTERN ErlDrvTid erl_drv_thread_self(void);
EXTERN int erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2);
EXTERN void erl_drv_thread_exit(void *resp);
EXTERN int erl_drv_thread_join(ErlDrvTid, void **respp);

EXTERN char* erl_drv_mutex_name(ErlDrvMutex *mtx);
EXTERN char* erl_drv_cond_name(ErlDrvCond *cnd);
EXTERN char* erl_drv_rwlock_name(ErlDrvRWLock *rwlck);
EXTERN char* erl_drv_thread_name(ErlDrvTid tid);

/*
 * Misc.
 */
EXTERN int null_func(void);

#endif /* !ERL_DRIVER_TYPES_ONLY */
# 539 "beam/erl_driver.h"

/* Constants for return flags from the 'port_call' callback */
#define DRIVER_CALL_KEEP_BUFFER 0x1

/* ErlDrvTerm is the type to use for casts when building 
 * terms that should be sent to connected process,
 * for instance a tuple on the form {tcp, Port, [Tag|Binary]}
 *
 * ErlDrvTerm spec[] = {
 *    ERL_DRV_ATOM, driver_mk_atom("tcp"),
 *    ERL_DRV_PORT, driver_mk_port(drv->ix),
 *             ERL_DRV_INT, REPLY_TAG,
 *             ERL_DRV_BINARY, (ErlDrvTerm)bin, 50, 0,
 *             ERL_DRV_LIST, 2,
 *    ERL_DRV_TUPLE, 3,
 *  }       
 *             
 */

#define TERM_DATA(x) ((ErlDrvTermData) (x))

/* Possible types to send from driver          Argument type */
#define ERL_DRV_NIL         ((ErlDrvTermData) 1)  /* None */
#define ERL_DRV_ATOM        ((ErlDrvTermData) 2)  /* driver_mk_atom(string) */
#define ERL_DRV_INT         ((ErlDrvTermData) 3)  /* ErlDrvSInt */
#define ERL_DRV_PORT        ((ErlDrvTermData) 4)  /* driver_mk_port(ix) */
#define ERL_DRV_BINARY      ((ErlDrvTermData) 5)  /* ErlDrvBinary*, 
						   * ErlDrvUInt size,
						   * ErlDrvUInt offs */
#define ERL_DRV_STRING      ((ErlDrvTermData) 6)  /* char*, ErlDrvUInt */
#define ERL_DRV_TUPLE       ((ErlDrvTermData) 7)  /* ErlDrvUInt */
#define ERL_DRV_LIST        ((ErlDrvTermData) 8)  /* ErlDrvUInt */
#define ERL_DRV_STRING_CONS ((ErlDrvTermData) 9)  /* char*, ErlDrvUInt */
#define ERL_DRV_PID         ((ErlDrvTermData) 10) /* driver_connected,... */

#define ERL_DRV_FLOAT       ((ErlDrvTermData) 11) /* double * */
#define ERL_DRV_EXT2TERM    ((ErlDrvTermData) 12) /* char *, ErlDrvUInt */
#define ERL_DRV_UINT        ((ErlDrvTermData) 13) /* ErlDrvUInt */
#define ERL_DRV_BUF2BINARY  ((ErlDrvTermData) 14) /* char *, ErlDrvUInt */
#define ERL_DRV_INT64       ((ErlDrvTermData) 15) /* ErlDrvSInt64 * */
#define ERL_DRV_UINT64      ((ErlDrvTermData) 16) /* ErlDrvUInt64 * */

#define ERL_DRV_MAP         ((ErlDrvTermData) 17) /* ErlDrvUInt */

#ifndef ERL_DRIVER_TYPES_ONLY

/* make terms for driver_output_term and driver_send_term */
EXTERN ErlDrvTermData driver_mk_atom(char*);
EXTERN ErlDrvTermData driver_mk_port(ErlDrvPort);
EXTERN ErlDrvTermData driver_connected(ErlDrvPort);
EXTERN ErlDrvTermData driver_caller(ErlDrvPort);
extern const ErlDrvTermData driver_term_nil;
EXTERN ErlDrvTermData driver_mk_term_nil(void);
EXTERN ErlDrvPort driver_create_port(ErlDrvPort creator_port, 
				     ErlDrvTermData connected, /* pid */
				     char* name, /* driver name */
				     ErlDrvData drv_data);
					 

/*
 * driver_output_term() is deprecated, and scheduled for removal in
 * OTP-R17. Use erl_drv_output_term() instead. For more information
 * see the erl_driver(3) documentation.
 */
EXTERN int driver_output_term(ErlDrvPort ix,
			      ErlDrvTermData* data,
			      int len) ERL_DRV_DEPRECATED_FUNC;
/*
 * driver_send_term() is deprecated, and scheduled for removal in
 * OTP-R17. Use erl_drv_send_term() instead. For more information
 * see the erl_driver(3) documentation.
 */
EXTERN int driver_send_term(ErlDrvPort ix,
			    ErlDrvTermData to,
			    ErlDrvTermData* data,
			    int len) ERL_DRV_DEPRECATED_FUNC;

/* output term data to the port owner */
EXTERN int erl_drv_output_term(ErlDrvTermData port,
			       ErlDrvTermData* data,
			       int len);
/* output term data to a specific process */
EXTERN int erl_drv_send_term(ErlDrvTermData port,
			     ErlDrvTermData to,
			     ErlDrvTermData* data,
			     int len);

/* Async IO functions */
EXTERN unsigned int driver_async_port_key(ErlDrvPort port);

EXTERN long driver_async(ErlDrvPort ix,
			 unsigned int* key,
			 void (*async_invoke)(void*), 
			 void* async_data,
			 void (*async_free)(void*));

/* Locks the driver in the machine "forever", there is
   no unlock function. Note that this is almost never useful, as an open
   port towards the driver locks it until the port is closed, why unexpected
   unloading "never" happens. */
EXTERN int driver_lock_driver(ErlDrvPort ix);

/* Get the current 'now' timestamp (analogue to erlang:now()) */
EXTERN int driver_get_now(ErlDrvNowData *now) ERL_DRV_DEPRECATED_FUNC;

/* Erlang Monotonic Time */
EXTERN ErlDrvTime erl_drv_monotonic_time(ErlDrvTimeUnit time_unit);
/* Time offset between Erlang Monotonic Time and Erlang System Time */
EXTERN ErlDrvTime erl_drv_time_offset(ErlDrvTimeUnit time_unit);
/* Time unit conversion */
EXTERN ErlDrvTime erl_drv_convert_time_unit(ErlDrvTime val,
					    ErlDrvTimeUnit from,
					    ErlDrvTimeUnit to);

/* These were removed from the ANSI version, now they're back. */

EXTERN void *driver_dl_open(char *);
EXTERN void *driver_dl_sym(void *, char *);
EXTERN int driver_dl_close(void *);
EXTERN char *driver_dl_error(void);

/* environment */
EXTERN int erl_drv_putenv(const char *key, char *value);
EXTERN int erl_drv_getenv(const char *key, char *value, size_t *value_size);

#ifdef __OSE__
typedef ErlDrvUInt ErlDrvOseEventId;
EXTERN union SIGNAL *erl_drv_ose_get_signal(ErlDrvEvent ev);
EXTERN ErlDrvEvent erl_drv_ose_event_alloc(SIGSELECT sig, ErlDrvOseEventId handle,
					   ErlDrvOseEventId (*resolve_signal)(union SIGNAL *sig), void *extra);
EXTERN void erl_drv_ose_event_free(ErlDrvEvent ev);
EXTERN void erl_drv_ose_event_fetch(ErlDrvEvent ev, SIGSELECT *sig,
                  ErlDrvOseEventId *handle, void **extra);
#endif
# 673 "beam/erl_driver.h"

#endif /* !ERL_DRIVER_TYPES_ONLY */
# 675 "beam/erl_driver.h"

#ifdef WIN32_DYNAMIC_ERL_DRIVER
#if 0 /* expanded by -frewrite-includes */
#  include "erl_win_dyn_driver.h"
#endif /* expanded by -frewrite-includes */
# 677 "beam/erl_driver.h"
# 678 "beam/erl_driver.h"
#endif
# 679 "beam/erl_driver.h"

#endif
# 681 "beam/erl_driver.h"

/* also in global.h, but driver's can't include global.h */
void dtrace_drvport_str(ErlDrvPort port, char *port_buf);
# 39 "beam/erl_sys_driver.h" 2

#endif
# 41 "beam/erl_sys_driver.h"




# 30 "beam/erl_port_task.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port_task.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 29 "beam/erl_smp.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 31 "beam/erl_port_task.h" 2
#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port_task.h"
# 1 "beam/erl_port.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_PORT_TYPE__
#define ERL_PORT_TYPE__
typedef struct _erl_drv_port Port;
typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
#endif
# 26 "beam/erl_port.h"

#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__)
#define ERL_PORT_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_port_task.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port.h"
# 31 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_ptab.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_port.h"
# 32 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port.h"
# 33 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_port.h"
# 34 "beam/erl_port.h"

#ifndef __WIN32__
#define ERTS_DEFAULT_MAX_PORTS (1 << 16)
#else
# 38 "beam/erl_port.h"
/* 
 *  Do not default to as many max ports on Windows
 *  as there are no os limits to stop system
 *  from running amok. If allowed to go too high
 *  windows rarely recovers from the errors and
 *  other OS processes can be effected. 
 */
#define ERTS_DEFAULT_MAX_PORTS (1 << 13)
#endif /* __WIN32__ */
# 47 "beam/erl_port.h"
#define ERTS_MIN_PORTS 1024

extern int erts_port_synchronous_ops;
extern int erts_port_schedule_all_ops;
extern int erts_port_parallelism;

typedef struct erts_driver_t_ erts_driver_t;

/*
 * It would have been preferred to use NULL as value of
 * ERTS_INVALID_ERL_DRV_PORT. That would, however, not be
 * backward compatible. In pre-R16 systems, 0 was a valid
 * port handle and -1 was used as invalid handle, so we
 * are stuck with it.
 */
#define ERTS_INVALID_ERL_DRV_PORT ((struct _erl_drv_port *) ((SWord) -1))
#ifdef DEBUG
/* Make sure we use this api, and do not cast directly */
#define ERTS_ErlDrvPort2Port(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((Port *) ((PH) - 4711)))
#define ERTS_Port2ErlDrvPort(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((ErlDrvPort) ((PH) + 4711)))
#else
# 74 "beam/erl_port.h"
#define ERTS_ErlDrvPort2Port(PH) ((Port *) (PH))
#define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH))
#endif
# 77 "beam/erl_port.h"

#define SMALL_IO_QUEUE 5   /* Number of fixed elements */

typedef struct {
    ErlDrvSizeT size;       /* total size in bytes */

    SysIOVec* v_start;
    SysIOVec* v_end;
    SysIOVec* v_head;
    SysIOVec* v_tail;
    SysIOVec  v_small[SMALL_IO_QUEUE];

    ErlDrvBinary** b_start;
    ErlDrvBinary** b_end;
    ErlDrvBinary** b_head;
    ErlDrvBinary** b_tail;
    ErlDrvBinary*  b_small[SMALL_IO_QUEUE];
} ErlIOQueue;

typedef struct line_buf {  /* Buffer used in line oriented I/O */
    ErlDrvSizeT bufsiz;      /* Size of character buffer */
    ErlDrvSizeT ovlen;       /* Length of overflow data */
    ErlDrvSizeT ovsiz;       /* Actual size of overflow buffer */
    char data[1];            /* Starting point of buffer data,
			      data[0] is a flag indicating an unprocess CR,
			      The rest is the overflow buffer. */
} LineBuf;

/*
 * Items part of erlang:port_info/1 result. Note am_registered_name
 * *need* to be first.
 */

#define ERTS_PORT_INFO_1_ITEMS				\
    {	am_registered_name,	/* Needs to be first */	\
	am_name,					\
	am_links,					\
	am_id,						\
	am_connected,					\
	am_input,					\
	am_output,					\
	am_os_pid }

/*
 * Port Specific Data.
 *
 * Only use PrtSD for very rarely used data.
 */

#define ERTS_PRTSD_SCHED_ID 0

#define ERTS_PRTSD_SIZE 1

typedef struct {
    void *data[ERTS_PRTSD_SIZE];
} ErtsPrtSD;

#ifdef ERTS_SMP
typedef struct ErtsXPortsList_ ErtsXPortsList;
#endif
# 137 "beam/erl_port.h"

/*
 * Port locking:
 *
 * Locking is done either driver specific or port specific. When
 * driver specific locking is used, all instances of the driver,
 * i.e. ports running the driver, share the same lock. When port
 * specific locking is used each instance have its own lock.
 *
 * Most fields in the Port structure are protected by the lock
 * referred to by the 'lock' field. This lock is shared between
 * all ports running the same driver when driver specific locking
 * is used.
 *
 * The 'sched' field is protected by the run queue lock that the
 * port currently is assigned to.
 *
 */

struct _erl_drv_port {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    ErtsPortTaskSched sched;
    ErtsPortTaskHandle timeout_task;
#ifdef ERTS_SMP
    erts_mtx_t *lock;
    ErtsXPortsList *xports;
    erts_smp_atomic_t run_queue;
#else
# 166 "beam/erl_port.h"
    erts_atomic32_t refc;
    int cleanup;
#endif
# 169 "beam/erl_port.h"
    erts_atomic_t connected;	/* A connected process */
    Eterm caller;		/* Current caller. */
    erts_smp_atomic_t data;	/* Data associated with port. */
    Uint bytes_in;		/* Number of bytes read */
    Uint bytes_out;		/* Number of bytes written */

    ErlIOQueue ioq;              /* driver accessible i/o queue */
    DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
    char *name;		         /* String used in the open */
    erts_driver_t* drv_ptr;
    UWord drv_data;
    SWord os_pid;                /* Child process ID */
    ErtsProcList *suspended;	 /* List of suspended processes. */
    LineBuf *linebuf;            /* Buffer to hold data not ready for
				    process to get (line oriented I/O)*/
    erts_atomic32_t state;	 /* Status and type flags */
    int control_flags;		 /* Flags for port_control()  */
    ErlDrvPDL port_data_lock;

    ErtsPrtSD *psd;		 /* Port specific data */
    int reds; /* Only used while executing driver callbacks */
};


void erts_init_port_data(Port *);
void erts_cleanup_port_data(Port *);
Uint erts_port_data_size(Port *);
ErlOffHeap *erts_port_data_offheap(Port *);

#define ERTS_PORT_GET_CONNECTED(PRT) \
    ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
    erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
    erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))


struct erl_drv_port_data_lock {
    erts_mtx_t mtx;
    erts_atomic_t refc;
    Port *prt;
};

ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsRunQueue *
erts_port_runq(Port *prt)
{
#ifdef ERTS_SMP
    ErtsRunQueue *rq1, *rq2;
    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
    if (!rq1)
	return NULL;
    while (1) {
	erts_smp_runq_lock(rq1);
	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
	if (rq1 == rq2)
	    return rq1;
	erts_smp_runq_unlock(rq1);
	rq1 = rq2;
	if (!rq1)
	    return NULL;
    }
#else
# 235 "beam/erl_port.h"
    return ERTS_RUNQ_IX(0);
#endif
# 237 "beam/erl_port.h"
}

#endif
# 240 "beam/erl_port.h"


ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix);
ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_prtsd_get(Port *prt, int ix)
{
    return prt->psd ? prt->psd->data[ix] : NULL;
}

ERTS_GLB_INLINE void *
erts_prtsd_set(Port *prt, int ix, void *data)
{
    if (prt->psd) {
	void *old = prt->psd->data[ix];
	prt->psd->data[ix] = data;
	return old;
    }
    else {
	prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
	prt->psd->data[ix] = data;
	return NULL;
    }
}

#endif
# 269 "beam/erl_port.h"

Eterm erts_request_io_bytes(Process *c_p);


/* port status flags */

#define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
/* Port have begun exiting */
#define ERTS_PORT_SFLG_EXITING		((Uint32) (1 <<  1))
/* Distribution port */
#define ERTS_PORT_SFLG_DISTRIBUTION	((Uint32) (1 <<  2))
#define ERTS_PORT_SFLG_BINARY_IO	((Uint32) (1 <<  3))
#define ERTS_PORT_SFLG_SOFT_EOF		((Uint32) (1 <<  4))
/* Flow control */
/* Port is closing (no i/o accepted) */
#define ERTS_PORT_SFLG_CLOSING		((Uint32) (1 <<  5))
/* Send a closed message when terminating */
#define ERTS_PORT_SFLG_SEND_CLOSED	((Uint32) (1 <<  6))
/* Line orinted io on port */  
#define ERTS_PORT_SFLG_LINEBUF_IO	((Uint32) (1 <<  7))
/* Immortal port (only certain system ports) */
#define ERTS_PORT_SFLG_FREE		((Uint32) (1 <<  8))
#define ERTS_PORT_SFLG_INITIALIZING	((Uint32) (1 <<  9))
/* Port uses port specific locking (opposed to driver specific locking) */
#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 10))
#define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 11))
/* Last port to terminate halts the emulator */
#define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 12))
#ifdef DEBUG
/* Only debug: make sure all flags aren't cleared unintentionally */
#define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
#endif
# 301 "beam/erl_port.h"

/* Combinations of port status flags */ 
#define ERTS_PORT_SFLGS_DEAD						\
  (ERTS_PORT_SFLG_FREE | ERTS_PORT_SFLG_INITIALIZING)
#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
  (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID)
#define ERTS_PORT_SFLGS_INVALID_LOOKUP					\
  (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
   | ERTS_PORT_SFLG_EXITING						\
   | ERTS_PORT_SFLG_CLOSING)
#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP				\
  (ERTS_PORT_SFLGS_INVALID_LOOKUP					\
   | ERTS_PORT_SFLG_DISTRIBUTION)

/*
 * Costs in reductions for some port operations.
 */
#define ERTS_PORT_REDS_EXECUTE		(CONTEXT_REDS/4)
#define ERTS_PORT_REDS_FREE		(CONTEXT_REDS/400)
#define ERTS_PORT_REDS_TIMEOUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_INPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_OUTPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EVENT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUTV	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUT	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EXIT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CONNECT		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_UNLINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_LINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_BADSIG		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_CONTROL		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CALL		(CONTEXT_REDS/50)
#define ERTS_PORT_REDS_INFO		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_TERMINATE	(CONTEXT_REDS/50)

void print_port_info(Port *, int, void *);
void erts_port_free(Port *);
#ifndef ERTS_SMP
void erts_port_cleanup(Port *);
#endif
# 341 "beam/erl_port.h"
void erts_fire_port_monitor(Port *prt, Eterm ref);
#ifdef ERTS_SMP
int erts_port_handle_xports(Port *);
#endif
# 345 "beam/erl_port.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_is_port_locked(Port *);
#endif
# 349 "beam/erl_port.h"

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);

ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt)
{
    erts_ptab_atmc_inc_refc(&prt->common);
}

ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt)
{
    int referred = erts_ptab_atmc_dec_test_refc(&prt->common);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc)
{
    int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
{
    return erts_ptab_atmc_read_refc(&prt->common);
}

ERTS_GLB_INLINE int
erts_smp_port_trylock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    return erts_mtx_trylock(prt->lock);
#else
# 393 "beam/erl_port.h"
    return 0;
#endif
# 395 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_lock(prt->lock);
#endif
# 405 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_unlock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_unlock(prt->lock);
#endif
# 415 "beam/erl_port.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 418 "beam/erl_port.h"


#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS)			\
    (!(PP)							\
     || (erts_atomic32_read_nob(&(PP)->state) & (FLGS))	\
     || (PP)->common.id != (ID))

/* port lookup */

#define INVALID_PORT(PP, ID) \
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP)

/* Invalidate trace port if anything suspicious, for instance
 * that the port is a distribution port or it is busy.
 */
#define INVALID_TRACER_PORT(PP, ID)					\
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)

#define ERTS_PORT_SCHED_ID(P, ID) \
  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))

extern const Port erts_invalid_port;
#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)

int erts_is_port_ioq_empty(Port *);
void erts_terminate_port(Port *);

#ifdef ERTS_SMP
Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
#endif
# 448 "beam/erl_port.h"

ERTS_GLB_INLINE Port *erts_pix2port(int);
ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm);
ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32);
ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
ERTS_GLB_INLINE void erts_port_release(Port *);
#ifdef ERTS_SMP
ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
#endif
# 459 "beam/erl_port.h"
ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int);
ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *);
ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort);
ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm);
ERTS_GLB_INLINE int erts_is_port_alive(Eterm);
ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm);
ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *);

#define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL)

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Port *erts_pix2port(int ix)
{
    Port *prt;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_port));
    prt = (Port *) erts_ptab_pix2intptr_nob(&erts_port, ix);
    return prt == ERTS_PORT_LOCK_BUSY ? NULL : prt;
}

ERTS_GLB_INLINE Port *
erts_port_lookup_raw(Eterm id)
{
    Port *prt;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));
    return prt && prt->common.id == id ? prt : NULL;
}

ERTS_GLB_INLINE Port *
erts_port_lookup(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt = erts_port_lookup_raw(id);
    return (!prt
	    ? NULL
	    : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state))
	       ? NULL
	       : prt));
}


ERTS_GLB_INLINE Port*
erts_id2port(Eterm id)
{
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

    erts_smp_port_lock(prt);
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) {
	erts_smp_port_unlock(prt);
	return NULL;
    }

    return prt;
}


ERTS_GLB_INLINE Port*
erts_id2port_sflgs(Eterm id,
		   Process *c_p, ErtsProcLocks c_p_locks,
		   Uint32 invalid_sflgs)
{
#ifdef ERTS_SMP
    int no_proc_locks = !c_p || !c_p_locks;
#endif
# 543 "beam/erl_port.h"
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

#ifdef ERTS_SMP
    if (no_proc_locks)
	erts_smp_port_lock(prt);
    else if (erts_smp_port_trylock(prt) == EBUSY) {
	/* Unlock process locks, and acquire locks in lock order... */
	erts_smp_proc_unlock(c_p, c_p_locks);
	erts_smp_port_lock(prt);
	erts_smp_proc_lock(c_p, c_p_locks);
    }
#endif
# 568 "beam/erl_port.h"
    state = erts_atomic32_read_nob(&prt->state);
    if (state & invalid_sflgs) {
#ifdef ERTS_SMP
	erts_smp_port_unlock(prt);
#endif
# 573 "beam/erl_port.h"
	return NULL;
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_port_release(Port *prt)
{
    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
#ifdef ERTS_SMP
    erts_smp_port_unlock(prt);
#else
# 587 "beam/erl_port.h"
    if (prt->cleanup) {
	prt->cleanup = 0;
	erts_port_cleanup(prt);
    }
#endif
# 592 "beam/erl_port.h"
}

#ifdef ERTS_SMP

/*
 * erts_thr_id2port_sflgs() and erts_thr_port_release() can
 * be used by unmanaged threads in the SMP case.
 */
ERTS_GLB_INLINE Port *
erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt;
    ErtsThrPrgrDelayHandle dhndl;

    if (is_not_internal_port(id))
	return NULL;

    dhndl = erts_thr_progress_unmanaged_delay();

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id) {
	erts_thr_progress_unmanaged_continue(dhndl);
	prt = NULL;
    }
    else {
	erts_aint32_t state;
	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
	    erts_port_inc_refc(prt);
	    erts_thr_progress_unmanaged_continue(dhndl);
	}

	erts_mtx_lock(prt->lock);
	state = erts_atomic32_read_nob(&prt->state);
	if (state & invalid_sflgs) {
	    erts_mtx_unlock(prt->lock);
	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
		erts_port_dec_refc(prt);
	    prt = NULL;
	}
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_thr_port_release(Port *prt)
{
    erts_mtx_unlock(prt->lock);
#ifdef ERTS_SMP
    if (!erts_thr_progress_is_managed_thread())
	erts_port_dec_refc(prt);
#endif
# 646 "beam/erl_port.h"
}

#endif
# 649 "beam/erl_port.h"

ERTS_GLB_INLINE Port *
erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    ASSERT(prt != NULL);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;

    if (lock_pdl && prt->port_data_lock)
	driver_pdl_lock(prt->port_data_lock);

#if ERTS_ENABLE_LOCK_CHECK
    if (!ERTS_IS_CRASH_DUMPING) {
	if (erts_lc_is_emu_thr()) {
	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	    ERTS_LC_ASSERT(!prt->port_data_lock
			   || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
	else {
	    ERTS_LC_ASSERT(prt->port_data_lock);
	    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
    }
#endif
# 674 "beam/erl_port.h"

    if (erts_atomic32_read_nob(&prt->state)
	& ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
	if (lock_pdl && prt->port_data_lock)
	    driver_pdl_unlock(prt->port_data_lock);
	return ERTS_INVALID_ERL_DRV_PORT;
    }
    return prt;
}

ERTS_GLB_INLINE Port *
erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    erts_aint32_t state;
    ASSERT(prt);
    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;
    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
		       || ERTS_IS_CRASH_DUMPING);
    /* 
     * This state check is only needed since a driver callback
     * might terminate the port, and then call back into the
     * emulator. Drivers should preferably have been forbidden
     * to call into the emulator after terminating the port,
     * but it has been like this for ages. Perhaps forbid this
     * in some future major release?
     */
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
	return ERTS_INVALID_ERL_DRV_PORT;
    if (statep)
	*statep = state;
    return prt;
}

ERTS_GLB_INLINE Eterm
erts_drvport2id(ErlDrvPort drvport)
{
    Port *prt = erts_drvport2port(drvport);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return am_undefined;
    else
	return prt->common.id;
}

ERTS_GLB_INLINE Uint32
erts_portid2status(Eterm id)
{
    Port *prt = erts_port_lookup_raw(id);
    if (prt)
	return (Uint32) erts_atomic32_read_acqb(&prt->state);
    else
	return ERTS_PORT_SFLG_INVALID;
}

ERTS_GLB_INLINE int
erts_is_port_alive(Eterm id)
{
    return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID
				       | ERTS_PORT_SFLGS_DEAD));
}

ERTS_GLB_INLINE int
erts_is_valid_tracer_port(Eterm id)
{
    return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
}

ERTS_GLB_INLINE int
erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
{
    int reds = 0;
    erts_aint32_t state;

    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));

    state = erts_atomic32_read_nob(&prt->state);
    if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) {
	reds += ERTS_PORT_REDS_TERMINATE;
	erts_terminate_port(prt);
	state = erts_atomic32_read_nob(&prt->state);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
    }

#ifdef ERTS_SMP
    if (prt->xports) {
	reds += erts_port_handle_xports(prt);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	ASSERT(!prt->xports);
    }
#endif
# 767 "beam/erl_port.h"

    if (statep)
	*statep = state;

    return reds;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 775 "beam/erl_port.h"

void erts_port_resume_procs(Port *);

struct binary;

#define ERTS_P2P_SIG_TYPE_BAD			0
#define ERTS_P2P_SIG_TYPE_OUTPUT		1
#define ERTS_P2P_SIG_TYPE_OUTPUTV		2
#define ERTS_P2P_SIG_TYPE_CONNECT		3
#define ERTS_P2P_SIG_TYPE_EXIT			4
#define ERTS_P2P_SIG_TYPE_CONTROL		5
#define ERTS_P2P_SIG_TYPE_CALL			6
#define ERTS_P2P_SIG_TYPE_INFO			7
#define ERTS_P2P_SIG_TYPE_LINK			8
#define ERTS_P2P_SIG_TYPE_UNLINK		9

#define ERTS_P2P_SIG_TYPE_BITS			4
#define ERTS_P2P_SIG_TYPE_MASK \
    ((1 << ERTS_P2P_SIG_TYPE_BITS) - 1)

#define ERTS_P2P_SIG_DATA_FLG(N) \
    (1 << (ERTS_P2P_SIG_TYPE_BITS + (N)))
#define ERTS_P2P_SIG_DATA_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG(0)
#define ERTS_P2P_SIG_DATA_FLG_REPLY		ERTS_P2P_SIG_DATA_FLG(1)
#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG(2)
#define ERTS_P2P_SIG_DATA_FLG_FORCE		ERTS_P2P_SIG_DATA_FLG(3)
#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT	ERTS_P2P_SIG_DATA_FLG(4)
#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK	ERTS_P2P_SIG_DATA_FLG(5)
#define ERTS_P2P_SIG_DATA_FLG_SCHED		ERTS_P2P_SIG_DATA_FLG(6)
#define ERTS_P2P_SIG_DATA_FLG_ASYNC		ERTS_P2P_SIG_DATA_FLG(7)

struct ErtsProc2PortSigData_ {
    int flags;
    Eterm caller;
    Uint32 ref[ERTS_MAX_REF_NUMBERS];
    union {
	struct {
	    Eterm from;
	    ErlIOVec *evp;
	    ErlDrvBinary *cbinp;
	} outputv;
	struct {
	    Eterm from;
	    char *bufp;
	    ErlDrvSizeT size;
	} output;
	struct {
	    Eterm from;
	    Eterm connected;
	} connect;
	struct {
	    Eterm from;
	    Eterm reason;
	    ErlHeapFragment *bp;
	} exit;
	struct {
	    struct binary *binp;
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} control;
	struct {
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} call;
	struct {
	    Eterm item;
	} info;
	struct {
	    Eterm port;
	    Eterm to;
	} link;
	struct {
	    Eterm from;
	} unlink;
    } u;
} ;

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp);
ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return !0;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return !0;
    default:				return 0;
    }
}

ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return sigdp->u.output.size;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return sigdp->u.outputv.evp->size;
    default:				return (ErlDrvSizeT) 0;
    }
}

#endif
# 882 "beam/erl_port.h"

#define ERTS_PROC2PORT_SIG_EXEC			0
#define ERTS_PROC2PORT_SIG_ABORT		1
#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND	2
#define ERTS_PROC2PORT_SIG_ABORT_CLOSED		3

typedef int (*ErtsProc2PortSigCallback)(Port *,
					erts_aint32_t,
					int,
					ErtsProc2PortSigData *);

typedef enum {
    ERTS_PORT_OP_BADARG,
    ERTS_PORT_OP_CALLER_EXIT,
    ERTS_PORT_OP_BUSY,
    ERTS_PORT_OP_BUSY_SCHEDULED,
    ERTS_PORT_OP_SCHEDULED,
    ERTS_PORT_OP_DROPPED,
    ERTS_PORT_OP_DONE
} ErtsPortOpResult;

ErtsPortOpResult
erts_schedule_proc2port_signal(Process *,
			       Port *,
			       Eterm,
			       Eterm *,
			       ErtsProc2PortSigData *,
			       int,
			       ErtsPortTaskHandle *,
			       ErtsProc2PortSigCallback);

int erts_deliver_port_exit(Port *, Eterm, Eterm, int);

/*
 * Port signal flags
 */
#define ERTS_PORT_SIG_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG_BANG_OP
#define ERTS_PORT_SIG_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG_NOSUSPEND
#define ERTS_PORT_SIG_FLG_FORCE			ERTS_P2P_SIG_DATA_FLG_FORCE
#define ERTS_PORT_SIG_FLG_BROKEN_LINK		ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK
#define ERTS_PORT_SIG_FLG_BAD_OUTPUT		ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT
#define ERTS_PORT_SIG_FLG_FORCE_SCHED		ERTS_P2P_SIG_DATA_FLG_SCHED
#define ERTS_PORT_SIG_FLG_ASYNC			ERTS_P2P_SIG_DATA_FLG_ASYNC
/* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */
#define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL	ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT

/*
 * Port ! {Owner, {command, Data}}
 * Port ! {Owner, {connect, NewOwner}}
 * Port ! {Owner, close}
 */
ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);

/*
 * Signals from processes to ports.
 */
ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);

#endif
# 948 "beam/erl_port.h"
# 33 "beam/erl_port_task.h" 2
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
typedef erts_smp_atomic_t ErtsPortTaskHandle;
#endif
# 36 "beam/erl_port_task.h"

#ifndef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
#ifndef ERL_PORT_TASK_H__
#define ERL_PORT_TASK_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || defined(ERL_IO_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 49 "beam/erl_port_task.h"

#define ERTS_PT_FLG_WAIT_BUSY		(1 << 0)
#define ERTS_PT_FLG_SIG_DEP		(1 << 1)
#define ERTS_PT_FLG_NOSUSPEND		(1 << 2)
#define ERTS_PT_FLG_REF			(1 << 3)
#define ERTS_PT_FLG_BAD_OUTPUT		(1 << 4)

typedef enum {
    ERTS_PORT_TASK_INPUT,
    ERTS_PORT_TASK_OUTPUT,
    ERTS_PORT_TASK_EVENT,
    ERTS_PORT_TASK_TIMEOUT,
    ERTS_PORT_TASK_DIST_CMD,
    ERTS_PORT_TASK_PROC_SIG
} ErtsPortTaskType;

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
/* NOTE: Do not access any of the exported variables directly */
extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
#endif
# 69 "beam/erl_port_task.h"

#define ERTS_PTS_FLG_IN_RUNQ			(((erts_aint32_t) 1) <<  0)
#define ERTS_PTS_FLG_EXEC			(((erts_aint32_t) 1) <<  1)
#define ERTS_PTS_FLG_HAVE_TASKS			(((erts_aint32_t) 1) <<  2)
#define ERTS_PTS_FLG_EXIT			(((erts_aint32_t) 1) <<  3)
#define ERTS_PTS_FLG_BUSY_PORT			(((erts_aint32_t) 1) <<  4)
#define ERTS_PTS_FLG_BUSY_PORT_Q		(((erts_aint32_t) 1) <<  5)
#define ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q	(((erts_aint32_t) 1) <<  6)
#define ERTS_PTS_FLG_HAVE_BUSY_TASKS		(((erts_aint32_t) 1) <<  7)
#define ERTS_PTS_FLG_HAVE_NS_TASKS		(((erts_aint32_t) 1) <<  8)
#define ERTS_PTS_FLG_PARALLELISM		(((erts_aint32_t) 1) <<  9)
#define ERTS_PTS_FLG_FORCE_SCHED		(((erts_aint32_t) 1) << 10)
#define ERTS_PTS_FLG_EXITING			(((erts_aint32_t) 1) << 11)
#define ERTS_PTS_FLG_EXEC_IMM			(((erts_aint32_t) 1) << 12)

#define ERTS_PTS_FLGS_BUSY \
    (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q)

#define ERTS_PTS_FLGS_FORCE_SCHEDULE_OP		\
    (ERTS_PTS_FLG_EXIT				\
     | ERTS_PTS_FLG_HAVE_BUSY_TASKS		\
     | ERTS_PTS_FLG_HAVE_TASKS			\
     | ERTS_PTS_FLG_EXEC			\
     | ERTS_PTS_FLG_EXEC_IMM			\
     | ERTS_PTS_FLG_FORCE_SCHED			\
     | ERTS_PTS_FLG_EXITING)

#define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH			8192
#define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW			4096

typedef struct {
    ErlDrvSizeT high;
    erts_smp_atomic_t low;
    erts_smp_atomic_t size;
}  ErtsPortTaskBusyPortQ;

typedef struct ErtsPortTask_ ErtsPortTask;
typedef struct ErtsPortTaskBusyCallerTable_ ErtsPortTaskBusyCallerTable;
typedef struct ErtsPortTaskHandleList_ ErtsPortTaskHandleList;

typedef struct {
    Port *next;
    struct {
	struct {
	    struct {
		ErtsPortTask *first;
		ErtsPortTask *last;
		ErtsPortTaskBusyCallerTable *table;
		ErtsPortTaskHandleList *nosuspend;
	    } busy;
	    ErtsPortTask *first;
	} local;
	struct {
	    ErtsPortTask *first;
	    ErtsPortTask *last;
	} in;
	ErtsPortTaskBusyPortQ *bpq;
    } taskq;
    erts_smp_atomic32_t flags;
#ifdef ERTS_SMP
    erts_mtx_t mtx;
#endif
# 131 "beam/erl_port_task.h"
} ErtsPortTaskSched;

ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp);
ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp);
ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
						   ErtsPortTaskBusyPortQ *bpq);
ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp,
					       Eterm id);
ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp);

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
#endif
# 148 "beam/erl_port_task.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
{
    erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL);
}

ERTS_GLB_INLINE int
erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
{
    return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL;
}

ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
						   ErtsPortTaskBusyPortQ *bpq)
{
    if (bpq) {
	erts_aint_t low = (erts_aint_t) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW;
	erts_smp_atomic_init_nob(&bpq->low, low);
 	bpq->high = (ErlDrvSizeT) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH;
	erts_smp_atomic_init_nob(&bpq->size, (erts_aint_t) 0);
    }
    ptsp->taskq.bpq = bpq;
}

ERTS_GLB_INLINE void
erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
{
#ifdef ERTS_SMP
    char *lock_str = "port_sched_lock";
#endif
# 181 "beam/erl_port_task.h"
    ptsp->next = NULL;
    ptsp->taskq.local.busy.first = NULL;
    ptsp->taskq.local.busy.last = NULL;
    ptsp->taskq.local.busy.table = NULL;
    ptsp->taskq.local.busy.nosuspend = NULL;
    ptsp->taskq.local.first = NULL;
    ptsp->taskq.in.first = NULL;
    ptsp->taskq.in.last = NULL;
    erts_smp_atomic32_init_nob(&ptsp->flags, 0);
#ifdef ERTS_SMP
    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id,
#ifdef ERTS_ENABLE_LOCK_COUNT
		    (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
#else
# 195 "beam/erl_port_task.h"
		    1
#endif
# 197 "beam/erl_port_task.h"
		    );
#endif
# 199 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE void
erts_port_task_sched_lock(ErtsPortTaskSched *ptsp)
{
#ifdef ERTS_SMP
    erts_mtx_lock(&ptsp->mtx);
#endif
# 207 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE void
erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(&ptsp->mtx);
#endif
# 215 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE int
erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(&ptsp->mtx);
#else
# 223 "beam/erl_port_task.h"
    return 0;
#endif
# 225 "beam/erl_port_task.h"
}


ERTS_GLB_INLINE void
erts_port_task_fini_sched(ErtsPortTaskSched *ptsp)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(&ptsp->mtx);
#endif
# 234 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE void
erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
{
    erts_smp_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
}

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

ERTS_GLB_INLINE int
erts_port_task_have_outstanding_io_tasks(void)
{
    return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
	    != 0);
}

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 252 "beam/erl_port_task.h"

#endif
# 254 "beam/erl_port_task.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
int erts_port_task_execute(ErtsRunQueue *, Port **);
void erts_port_task_init(void);
#endif
# 259 "beam/erl_port_task.h"

void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *);
int erts_port_task_abort(ErtsPortTaskHandle *);

void erts_port_task_abort_nosuspend_tasks(Port *);

int erts_port_task_schedule(Eterm,
			    ErtsPortTaskHandle *,
			    ErtsPortTaskType,
			    ...);
void erts_port_task_free_port(Port *);
int erts_port_is_scheduled(Port *);
ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void);

#ifdef ERTS_SMP
void erts_enqueue_port(ErtsRunQueue *rq, Port *pp);
Port *erts_dequeue_port(ErtsRunQueue *rq);
#endif
# 277 "beam/erl_port_task.h"
#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif /* ERL_PORT_TASK_H__ */
# 279 "beam/erl_port_task.h"
#endif /* ERTS_PORT_TASK_ONLY_BASIC_TYPES__ */
# 280 "beam/erl_port_task.h"
# 49 "beam/erl_node_tables.h" 2
#undef ERTS_PORT_TASK_ONLY_BASIC_TYPES__

#define ERTS_NODE_TAB_DELAY_GC_DEFAULT (60)
#define ERTS_NODE_TAB_DELAY_GC_MAX (100*1000*1000)
#define ERTS_NODE_TAB_DELAY_GC_INFINITY (ERTS_NODE_TAB_DELAY_GC_MAX+1)
 
#define ERST_INTERNAL_CHANNEL_NO 0

#define ERTS_DE_SFLG_CONNECTED			(((Uint32) 1) <<  0)
#define ERTS_DE_SFLG_EXITING			(((Uint32) 1) <<  1)

#define ERTS_DE_SFLGS_ALL			(ERTS_DE_SFLG_CONNECTED \
						 | ERTS_DE_SFLG_EXITING)

#define ERTS_DE_QFLG_BUSY			(((Uint32) 1) <<  0)
#define ERTS_DE_QFLG_EXIT			(((Uint32) 1) <<  1)

#define ERTS_DE_QFLGS_ALL			(ERTS_DE_QFLG_BUSY \
						 | ERTS_DE_QFLG_EXIT)

#if defined(ARCH_64) && !HALFWORD_HEAP
#define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713f713f713UL)
#else
# 72 "beam/erl_node_tables.h"
#define ERTS_DIST_OUTPUT_BUF_DBG_PATTERN ((Uint) 0xf713f713)
#endif
# 74 "beam/erl_node_tables.h"

typedef struct ErtsDistOutputBuf_ ErtsDistOutputBuf;
struct ErtsDistOutputBuf_ {
#ifdef DEBUG
    Uint dbg_pattern;
#endif
# 80 "beam/erl_node_tables.h"
    ErtsDistOutputBuf *next;
    byte *extp;
    byte *ext_endp;
    byte data[1];
};

typedef struct {
    ErtsDistOutputBuf *first;
    ErtsDistOutputBuf *last;
} ErtsDistOutputQueue;

struct ErtsProcList_;

/*
 * Lock order:
 *   1. dist_entry->rwmtx
 *   2. erts_node_table_rwmtx
 *   3. erts_dist_table_rwmtx
 *
 *   Lock mutexes with lower numbers before mutexes with higher numbers and
 *   unlock mutexes with higher numbers before mutexes with higher numbers.
 */

struct erl_link;

typedef struct dist_entry_ {
    HashBucket hash_bucket;     /* Hash bucket */
    struct dist_entry_ *next;	/* Next entry in dist_table (not sorted) */
    struct dist_entry_ *prev;	/* Previous entry in dist_table (not sorted) */
    erts_refc_t refc;		/* Reference count */

    erts_smp_rwmtx_t rwmtx;     /* Protects all fields below until lck_mtx. */
    Eterm sysname;		/* name@host atom for efficiency */
    Uint32 creation;		/* creation of connected node */
    Eterm cid;			/* connection handler (pid or port), NIL == free */
    Uint32 connection_id;	/* Connection id incremented on connect */
    Uint32 status;		/* Slot status, like exiting reserved etc */
    Uint32 flags;		/* Distribution flags, like hidden, 
				   atom cache etc. */
    unsigned long version;	/* Protocol version */


    erts_smp_mtx_t lnk_mtx;     /* Protects node_links, nlinks, and
				   monitors. */
    ErtsLink *node_links;       /* In a dist entry, node links are kept 
				   in a separate tree, while they are 
				   colocted with the ordinary link tree
				   for processes. It's not due to confusion,
				   it's because the link tree for the dist 
				   entry is in two levels, see erl_monitors.h 
				*/
    ErtsLink *nlinks;           /* Link tree with subtrees */
    ErtsMonitor *monitors;      /* Monitor tree */

    erts_smp_mtx_t qlock;       /* Protects qflgs and out_queue */
    Uint32 qflgs;
    Sint qsize;
    ErtsDistOutputQueue out_queue;
    struct ErtsProcList_ *suspended;

    ErtsDistOutputQueue finalized_out_queue;
    erts_smp_atomic_t dist_cmd_scheduled;
    ErtsPortTaskHandle dist_cmd;

    Uint (*send)(Port *prt, ErtsDistOutputBuf *obuf);

    struct cache* cache;	/* The atom cache */
} DistEntry;

typedef struct erl_node_ {
  HashBucket hash_bucket;	/* Hash bucket */
  erts_refc_t refc;		/* Reference count */
  Eterm	sysname;		/* name@host atom for efficiency */
  Uint32 creation;		/* Creation */
  DistEntry *dist_entry;	/* Corresponding dist entry */
} ErlNode;


extern Hash erts_dist_table;
extern Hash erts_node_table;
extern erts_smp_rwmtx_t erts_dist_table_rwmtx;
extern erts_smp_rwmtx_t erts_node_table_rwmtx;

extern DistEntry *erts_hidden_dist_entries;
extern DistEntry *erts_visible_dist_entries;
extern DistEntry *erts_not_connected_dist_entries;
extern Sint erts_no_of_hidden_dist_entries;
extern Sint erts_no_of_visible_dist_entries;
extern Sint erts_no_of_not_connected_dist_entries;

extern DistEntry *erts_this_dist_entry;
extern ErlNode *erts_this_node;
extern char *erts_this_node_sysname; /* must match erl_node_tables.c */

Uint erts_delayed_node_table_gc(void);
DistEntry *erts_channel_no_to_dist_entry(Uint);
DistEntry *erts_sysname_to_connected_dist_entry(Eterm);
DistEntry *erts_find_or_insert_dist_entry(Eterm);
DistEntry *erts_find_dist_entry(Eterm);
void erts_schedule_delete_dist_entry(DistEntry *);
Uint erts_dist_table_size(void);
void erts_dist_table_info(int, void *);
void erts_set_dist_entry_not_connected(DistEntry *);
void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint);
ErlNode *erts_find_or_insert_node(Eterm, Uint);
void erts_schedule_delete_node(ErlNode *);
void erts_set_this_node(Eterm, Uint);
Uint erts_node_table_size(void);
void erts_init_node_tables(int);
void erts_node_table_info(int, void *);
void erts_print_node_info(int, void *, Eterm, int*, int*);
Eterm erts_get_node_and_dist_references(struct process *);
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_is_de_rwlocked(DistEntry *);
int erts_lc_is_de_rlocked(DistEntry *);
#endif
# 196 "beam/erl_node_tables.h"

ERTS_GLB_INLINE void erts_deref_dist_entry(DistEntry *dep);
ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np);
ERTS_GLB_INLINE void erts_smp_de_rlock(DistEntry *dep);
ERTS_GLB_INLINE void erts_smp_de_runlock(DistEntry *dep);
ERTS_GLB_INLINE void erts_smp_de_rwlock(DistEntry *dep);
ERTS_GLB_INLINE void erts_smp_de_rwunlock(DistEntry *dep);
ERTS_GLB_INLINE void erts_smp_de_links_lock(DistEntry *dep);
ERTS_GLB_INLINE void erts_smp_de_links_unlock(DistEntry *dep);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_deref_dist_entry(DistEntry *dep)
{
    ASSERT(dep);
    if (erts_refc_dectest(&dep->refc, 0) == 0)
	erts_schedule_delete_dist_entry(dep);
}

ERTS_GLB_INLINE void
erts_deref_node_entry(ErlNode *np)
{
    ASSERT(np);
    if (erts_refc_dectest(&np->refc, 0) == 0)
	erts_schedule_delete_node(np);
}

ERTS_GLB_INLINE void
erts_smp_de_rlock(DistEntry *dep)
{
    erts_smp_rwmtx_rlock(&dep->rwmtx);
}

ERTS_GLB_INLINE void
erts_smp_de_runlock(DistEntry *dep)
{
    erts_smp_rwmtx_runlock(&dep->rwmtx);
}

ERTS_GLB_INLINE void
erts_smp_de_rwlock(DistEntry *dep)
{
    erts_smp_rwmtx_rwlock(&dep->rwmtx);
}

ERTS_GLB_INLINE void
erts_smp_de_rwunlock(DistEntry *dep)
{
    erts_smp_rwmtx_rwunlock(&dep->rwmtx);
}

ERTS_GLB_INLINE void
erts_smp_de_links_lock(DistEntry *dep)
{
    erts_smp_mtx_lock(&dep->lnk_mtx);
}

ERTS_GLB_INLINE void
erts_smp_de_links_unlock(DistEntry *dep)
{
    erts_smp_mtx_unlock(&dep->lnk_mtx);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 261 "beam/erl_node_tables.h"

void erts_debug_test_node_tab_delayed_delete(Sint64 millisecs);

#endif
# 265 "beam/erl_node_tables.h"
# 86 "beam/external.h" 2

#define ERTS_ATOM_CACHE_SIZE 2048

typedef struct cache {
    Eterm in_arr[ERTS_ATOM_CACHE_SIZE];
    Eterm out_arr[ERTS_ATOM_CACHE_SIZE];
} ErtsAtomCache;

typedef struct {
    int hdr_sz;
    int sz;
    int long_atoms;
    int cix[ERTS_ATOM_CACHE_SIZE];
    struct {
	Eterm atom;
	int iix;
    } cache[ERTS_ATOM_CACHE_SIZE];
} ErtsAtomCacheMap;

typedef struct {
    Uint32 size;
    Eterm atom[ERTS_ATOM_CACHE_SIZE];
} ErtsAtomTranslationTable;

/*
 * These flags are tagged onto the high bits of a connection ID and stored in
 * the ErtsDistExternal structure's flags field.  They are used to indicate
 * various bits of state necessary to decode binaries in a variety of
 * scenarios. The mask ERTS_DIST_EXT_CON_ID_MASK is used later to separate the
 * connection ID from the flags. Be careful to ensure that the mask does not
 * overlap any of the bits used for flags, or ERTS will leak flags bits into
 * connection IDs and leak connection ID bits into the flags.
 */
#define ERTS_DIST_EXT_DFLAG_HDR      ((Uint32) 0x80000000)
#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x40000000)
#define ERTS_DIST_EXT_BTT_SAFE       ((Uint32) 0x20000000)
#define ERTS_DIST_EXT_CON_ID_MASK    ((Uint32) 0x1fffffff)

#define ERTS_DIST_EXT_CON_ID(DIST_EXTP) \
  ((DIST_EXTP)->flags & ERTS_DIST_EXT_CON_ID_MASK)
typedef struct {
    DistEntry *dep;
    byte *extp;
    byte *ext_endp;
    Sint heap_size;
    Uint32 flags;
    ErtsAtomTranslationTable attab;
} ErtsDistExternal;

typedef struct {
    int have_header;
    int cache_entries;
} ErtsDistHeaderPeek;

#define ERTS_DIST_EXT_SIZE(EDEP) \
  (sizeof(ErtsDistExternal) \
   - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \
      ? (ASSERT(0 <= (EDEP)->attab.size \
		&& (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \
	 sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - (EDEP)->attab.size)) \
      : sizeof(ErtsAtomTranslationTable)))

typedef struct {
    byte *extp;
    int exttmp;
    Uint extsize;
    Uint heap_size;
} ErtsBinary2TermState;


/* -------------------------------------------------------------------------- */

void erts_init_atom_cache_map(ErtsAtomCacheMap *);
void erts_reset_atom_cache_map(ErtsAtomCacheMap *);
void erts_destroy_atom_cache_map(ErtsAtomCacheMap *);
void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32);

Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *);
byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32);
struct erts_dsig_send_context;
int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
struct TTBEncodeContext_;
int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *,
			  struct TTBEncodeContext_ *, Sint* reds);

Uint erts_encode_ext_size(Eterm);
Uint erts_encode_ext_size_2(Eterm, unsigned);
Uint erts_encode_ext_size_ets(Eterm);
void erts_encode_ext(Eterm, byte **);
byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap);

#ifdef ERTS_WANT_EXTERNAL_TAGS
ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint);
#endif
# 182 "beam/external.h"
ERTS_GLB_INLINE void erts_free_dist_ext_copy(ErtsDistExternal *);
ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *);
ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint);
void *erts_dist_ext_trailer(ErtsDistExternal *);
void erts_destroy_dist_ext_copy(ErtsDistExternal *);
int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
			  DistEntry *, ErtsAtomCache *);
Sint erts_decode_dist_ext_size(ErtsDistExternal *);
Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);

Sint erts_decode_ext_size(byte*, Uint);
Sint erts_decode_ext_size_ets(byte*, Uint);
Eterm erts_decode_ext(ErtsHeapFactory*, byte**);
Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*);

Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);

Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint);
void erts_binary2term_abort(ErtsBinary2TermState *);
Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*);
int erts_debug_max_atom_out_cache_index(void);
int erts_debug_atom_to_out_cache_index(Eterm);


#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#ifdef ERTS_WANT_EXTERNAL_TAGS
ERTS_GLB_INLINE void
erts_peek_dist_header(ErtsDistHeaderPeek *dhpp, byte *ext, Uint sz)
{
    if (ext[0] == VERSION_MAGIC
	|| ext[1] != DIST_HEADER
	|| sz < (1+1+1))
	dhpp->have_header = 0;
    else {
	dhpp->have_header = 1;
	dhpp->cache_entries = (int) get_int8(&ext[2]);
    }
}
#endif
# 221 "beam/external.h"

ERTS_GLB_INLINE void
erts_free_dist_ext_copy(ErtsDistExternal *edep)
{
    if (edep->dep)
	erts_deref_dist_entry(edep->dep);
    erts_free(ERTS_ALC_T_EXT_TERM_DATA, edep);
}

ERTS_GLB_INLINE void *
erts_dist_ext_trailer(ErtsDistExternal *edep)
{
    void *res = (void *) (edep->ext_endp
			  + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp));
    ASSERT((((UWord) res) % sizeof(Uint)) == 0);
    return res;
}

#endif
# 240 "beam/external.h"

#endif /* ERL_EXTERNAL_H__ */
# 242 "beam/external.h"
# 97 "beam/erl_message.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 97 "beam/erl_message.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 98 "beam/erl_message.h" 2

/*
 * This struct represents a heap fragment, which is used when there
 * isn't sufficient room in the process heap and we can't do a GC.
 */

typedef struct erl_heap_fragment ErlHeapFragment;
struct erl_heap_fragment {
    ErlHeapFragment* next;	/* Next heap fragment */
    ErlOffHeap off_heap;	/* Offset heap data. */
    unsigned alloc_size;	/* Size in (half)words of mem */
    unsigned used_size;         /* With terms to be moved to heap by GC */
    Eterm mem[1];		/* Data */
};

typedef struct erl_mesg {
    struct erl_mesg* next;	/* Next message */
    union {
	ErtsDistExternal *dist_ext;
	ErlHeapFragment *heap_frag;
	void *attached;
    } data;
#ifdef USE_VM_PROBES
    Eterm m[3];                 /* m[0] = message, m[1] = seq trace token, m[3] = dynamic trace user tag */
#else
# 123 "beam/erl_message.h"
    Eterm m[2];			/* m[0] = message, m[1] = seq trace token */
#endif
# 125 "beam/erl_message.h"
} ErlMessage;

#define ERL_MESSAGE_TERM(mp) ((mp)->m[0])
#define ERL_MESSAGE_TOKEN(mp) ((mp)->m[1])
#ifdef USE_VM_PROBES
#define ERL_MESSAGE_DT_UTAG(mp) ((mp)->m[2])
#endif
# 132 "beam/erl_message.h"

/* Size of default message buffer (erl_message.c) */
#define ERL_MESSAGE_BUF_SZ 500

typedef struct {
    ErlMessage* first;
    ErlMessage** last;  /* point to the last next pointer */
    ErlMessage** save;
    Sint len;            /* queue length */

    /*
     * The following two fields are used by the recv_mark/1 and
     * recv_set/1 instructions.
     */
    BeamInstr* mark;		/* address to rec_loop/2 instruction */
    ErlMessage** saved_last;	/* saved last pointer */
} ErlMessageQueue;

#ifdef ERTS_SMP

typedef struct {
    ErlMessage* first;
    ErlMessage** last;  /* point to the last next pointer */
    Sint len;            /* queue length */
} ErlMessageInQueue;

#endif
# 159 "beam/erl_message.h"

/* Get "current" message */
#define PEEK_MESSAGE(p)  (*(p)->msg.save)


/* Add message last in private message queue */
#define LINK_MESSAGE_PRIVQ(p, mp) do { \
    *(p)->msg.last = (mp); \
    (p)->msg.last = &(mp)->next; \
    (p)->msg.len++; \
} while (0)


#ifdef ERTS_SMP

/* Move in message queue to end of private message queue */
#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P)					\
do {									\
    if ((P)->msg_inq.first) {						\
	*(P)->msg.last = (P)->msg_inq.first;				\
	(P)->msg.last = (P)->msg_inq.last;				\
	(P)->msg.len += (P)->msg_inq.len;				\
	(P)->msg_inq.first = NULL;					\
	(P)->msg_inq.last = &(P)->msg_inq.first;			\
	(P)->msg_inq.len = 0;						\
    }									\
} while (0)

/* Add message last in message queue */
#define LINK_MESSAGE(p, mp) do { \
    *(p)->msg_inq.last = (mp); \
    (p)->msg_inq.last = &(mp)->next; \
    (p)->msg_inq.len++; \
} while(0)

#else
# 195 "beam/erl_message.h"

#define ERTS_SMP_MSGQ_MV_INQ2PRIVQ(P)

/* Add message last in message queue */
#define LINK_MESSAGE(p, mp) LINK_MESSAGE_PRIVQ((p), (mp))

#endif
# 202 "beam/erl_message.h"

/* Unlink current message */
#define UNLINK_MESSAGE(p,msgp) do { \
     ErlMessage* __mp = (msgp)->next; \
     *(p)->msg.save = __mp; \
     (p)->msg.len--; \
     if (__mp == NULL) \
         (p)->msg.last = (p)->msg.save; \
     (p)->msg.mark = 0; \
} while(0)

/* Reset message save point (after receive match) */
#define JOIN_MESSAGE(p) \
     (p)->msg.save = &(p)->msg.first

/* Save current message */
#define SAVE_MESSAGE(p) \
     (p)->msg.save = &(*(p)->msg.save)->next

/*
 * ErtsMoveMsgAttachmentIntoProc() moves data attached to a message
 * onto the heap of a process. The attached data is the content of
 * the the message either on the internal format or on the external
 * format, and also possibly a seq trace token on the internal format.
 * If the message content is on the external format, the decode might
 * fail. If the decoding fails, ERL_MESSAGE_TERM(M) will contain
 * THE_NON_VALUE. That is, ERL_MESSAGE_TERM(M) *has* to be checked
 * afterwards and taken care of appropriately.
 *
 * ErtsMoveMsgAttachmentIntoProc() will shallow copy to heap if
 * possible; otherwise, move to heap via garbage collection.
 *
 * ErtsMoveMsgAttachmentIntoProc() is used when receiveing messages
 * in process_main() and in hipe_check_get_msg().
 */

#define ErtsMoveMsgAttachmentIntoProc(M, P, ST, HT, FC, SWPO, SWPI)	\
do {									\
    if ((M)->data.attached) {						\
	Uint need__ = erts_msg_attached_data_size((M));			\
        { SWPO ; }							\
 	if ((ST) - (HT) >= need__) {					\
            ErtsHeapFactory factory__;                                  \
            erts_factory_proc_prealloc_init(&factory__, (P), need__);   \
	    erts_move_msg_attached_data_to_heap(&factory__, (M));       \
            erts_factory_close(&factory__);                             \
            if ((P)->mbuf != NULL) {                                    \
                /* Heap was exhausted by messages. This is a rare case */   \
                /* that can currently (OTP 18) only happen if hamts are */  \
                /* far exceeding the estimated heap size. Do GC. */         \
                (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
            }								\
	}								\
	else {								\
	    (FC) -= erts_garbage_collect((P), 0, NULL, 0);		\
	}								\
        { SWPI ; }							\
	ASSERT(!(M)->data.attached);					\
    }									\
} while (0)

#define ERTS_SND_FLG_NO_SEQ_TRACE		(((unsigned) 1) << 0)

#define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \
   (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm))

#define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS)	\
do {							\
    (HEAP_FRAG_P)->next = NULL;				\
    (HEAP_FRAG_P)->alloc_size = (DATA_WORDS);		\
    (HEAP_FRAG_P)->used_size = (DATA_WORDS);            \
    (HEAP_FRAG_P)->off_heap.first = NULL; 	        \
    (HEAP_FRAG_P)->off_heap.overhead = 0;		\
} while (0)

void init_message(void);
void free_message(ErlMessage *);
ErlHeapFragment* new_message_buffer(Uint);
ErlHeapFragment* erts_resize_message_buffer(ErlHeapFragment *, Uint,
					    Eterm *, Uint);
void free_message_buffer(ErlHeapFragment *);
void erts_queue_dist_message(Process*, ErtsProcLocks*, ErtsDistExternal *, Eterm);
#ifdef USE_VM_PROBES
void erts_queue_message_probe(Process*, ErtsProcLocks*, ErlHeapFragment*,
                              Eterm message, Eterm seq_trace_token, Eterm dt_utag);
#define erts_queue_message(RP,RL,BP,Msg,SEQ) \
    erts_queue_message_probe((RP),(RL),(BP),(Msg),(SEQ),NIL)
#else
# 290 "beam/erl_message.h"
void erts_queue_message(Process*, ErtsProcLocks*, ErlHeapFragment*,
                        Eterm message, Eterm seq_trace_token);
#define erts_queue_message_probe(RP,RL,BP,Msg,SEQ,TAG) \
    erts_queue_message((RP),(RL),(BP),(Msg),(SEQ))
#endif
# 295 "beam/erl_message.h"
void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm);
Sint erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned);
void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp);

void erts_move_msg_mbuf_to_heap(Eterm**, ErlOffHeap*, ErlMessage *);

Uint erts_msg_attached_data_size_aux(ErlMessage *msg);
void erts_move_msg_attached_data_to_heap(ErtsHeapFactory*, ErlMessage *);
Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **,
			    Eterm *, ErtsDistExternal *);

void erts_cleanup_offheap(ErlOffHeap *offheap);


ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment*);
ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Uint erts_used_frag_sz(const ErlHeapFragment* bp)
{
    Uint sz = 0;
    for ( ; bp!=NULL; bp=bp->next) {
	sz += bp->used_size;
    }
    return sz;
}

ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg)
{
    ASSERT(msg->data.attached);
    if (is_value(ERL_MESSAGE_TERM(msg)))
	return erts_used_frag_sz(msg->data.heap_frag);
    else if (msg->data.dist_ext->heap_size < 0)
	return erts_msg_attached_data_size_aux(msg);
    else {
	Uint sz = msg->data.dist_ext->heap_size;
	if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) {
	    ErlHeapFragment *heap_frag;
	    heap_frag = erts_dist_ext_trailer(msg->data.dist_ext);
	    sz += heap_frag->used_size;
	}
	return sz;
    }
}
#endif
# 340 "beam/erl_message.h"

#endif
# 342 "beam/erl_message.h"
# 52 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 1 "beam/erl_process_dict.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 1999-2009. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef _ERL_PROCESS_DICT_H
#define _ERL_PROCESS_DICT_H
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 23 "beam/erl_process_dict.h"
# 24 "beam/erl_process_dict.h"

typedef struct proc_dict {
    unsigned int size;
    unsigned int used;
    unsigned int homeSize;
    unsigned int splitPosition;
    Uint numElements;
    Eterm data[1]; /* The beginning of an array of erlang terms */
} ProcDict;

Uint erts_dicts_mem_size(struct process *p);
void erts_erase_dicts(struct process *p);
void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd);
void erts_deep_dictionary_dump(int to, void *to_arg,
			       ProcDict* pd, void (*cb)(int, void *, Eterm obj));
Eterm erts_dictionary_copy(struct process *p, ProcDict *pd);

Eterm erts_pd_hash_get(struct process *p, Eterm id);

#endif
# 44 "beam/erl_process_dict.h"
# 53 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 1 "beam/erl_hl_timer.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2015. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef ERL_HL_TIMER_H__
#define ERL_HL_TIMER_H__

typedef struct ErtsHLTimer_ ErtsBifTimers;
typedef struct ErtsHLTimerService_ ErtsHLTimerService;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/erl_hl_timer.h"
# 28 "beam/erl_hl_timer.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_hl_timer.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 29 "beam/erl_hl_timer.h" 2
#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_hl_timer.h"
# 1 "beam/erl_port.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_PORT_TYPE__
#define ERL_PORT_TYPE__
typedef struct _erl_drv_port Port;
typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
#endif
# 26 "beam/erl_port.h"

#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__)
#define ERL_PORT_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_port_task.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port.h"
# 31 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_ptab.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_port.h"
# 32 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port.h"
# 33 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_port.h"
# 34 "beam/erl_port.h"

#ifndef __WIN32__
#define ERTS_DEFAULT_MAX_PORTS (1 << 16)
#else
# 38 "beam/erl_port.h"
/* 
 *  Do not default to as many max ports on Windows
 *  as there are no os limits to stop system
 *  from running amok. If allowed to go too high
 *  windows rarely recovers from the errors and
 *  other OS processes can be effected. 
 */
#define ERTS_DEFAULT_MAX_PORTS (1 << 13)
#endif /* __WIN32__ */
# 47 "beam/erl_port.h"
#define ERTS_MIN_PORTS 1024

extern int erts_port_synchronous_ops;
extern int erts_port_schedule_all_ops;
extern int erts_port_parallelism;

typedef struct erts_driver_t_ erts_driver_t;

/*
 * It would have been preferred to use NULL as value of
 * ERTS_INVALID_ERL_DRV_PORT. That would, however, not be
 * backward compatible. In pre-R16 systems, 0 was a valid
 * port handle and -1 was used as invalid handle, so we
 * are stuck with it.
 */
#define ERTS_INVALID_ERL_DRV_PORT ((struct _erl_drv_port *) ((SWord) -1))
#ifdef DEBUG
/* Make sure we use this api, and do not cast directly */
#define ERTS_ErlDrvPort2Port(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((Port *) ((PH) - 4711)))
#define ERTS_Port2ErlDrvPort(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((ErlDrvPort) ((PH) + 4711)))
#else
# 74 "beam/erl_port.h"
#define ERTS_ErlDrvPort2Port(PH) ((Port *) (PH))
#define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH))
#endif
# 77 "beam/erl_port.h"

#define SMALL_IO_QUEUE 5   /* Number of fixed elements */

typedef struct {
    ErlDrvSizeT size;       /* total size in bytes */

    SysIOVec* v_start;
    SysIOVec* v_end;
    SysIOVec* v_head;
    SysIOVec* v_tail;
    SysIOVec  v_small[SMALL_IO_QUEUE];

    ErlDrvBinary** b_start;
    ErlDrvBinary** b_end;
    ErlDrvBinary** b_head;
    ErlDrvBinary** b_tail;
    ErlDrvBinary*  b_small[SMALL_IO_QUEUE];
} ErlIOQueue;

typedef struct line_buf {  /* Buffer used in line oriented I/O */
    ErlDrvSizeT bufsiz;      /* Size of character buffer */
    ErlDrvSizeT ovlen;       /* Length of overflow data */
    ErlDrvSizeT ovsiz;       /* Actual size of overflow buffer */
    char data[1];            /* Starting point of buffer data,
			      data[0] is a flag indicating an unprocess CR,
			      The rest is the overflow buffer. */
} LineBuf;

/*
 * Items part of erlang:port_info/1 result. Note am_registered_name
 * *need* to be first.
 */

#define ERTS_PORT_INFO_1_ITEMS				\
    {	am_registered_name,	/* Needs to be first */	\
	am_name,					\
	am_links,					\
	am_id,						\
	am_connected,					\
	am_input,					\
	am_output,					\
	am_os_pid }

/*
 * Port Specific Data.
 *
 * Only use PrtSD for very rarely used data.
 */

#define ERTS_PRTSD_SCHED_ID 0

#define ERTS_PRTSD_SIZE 1

typedef struct {
    void *data[ERTS_PRTSD_SIZE];
} ErtsPrtSD;

#ifdef ERTS_SMP
typedef struct ErtsXPortsList_ ErtsXPortsList;
#endif
# 137 "beam/erl_port.h"

/*
 * Port locking:
 *
 * Locking is done either driver specific or port specific. When
 * driver specific locking is used, all instances of the driver,
 * i.e. ports running the driver, share the same lock. When port
 * specific locking is used each instance have its own lock.
 *
 * Most fields in the Port structure are protected by the lock
 * referred to by the 'lock' field. This lock is shared between
 * all ports running the same driver when driver specific locking
 * is used.
 *
 * The 'sched' field is protected by the run queue lock that the
 * port currently is assigned to.
 *
 */

struct _erl_drv_port {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    ErtsPortTaskSched sched;
    ErtsPortTaskHandle timeout_task;
#ifdef ERTS_SMP
    erts_mtx_t *lock;
    ErtsXPortsList *xports;
    erts_smp_atomic_t run_queue;
#else
# 166 "beam/erl_port.h"
    erts_atomic32_t refc;
    int cleanup;
#endif
# 169 "beam/erl_port.h"
    erts_atomic_t connected;	/* A connected process */
    Eterm caller;		/* Current caller. */
    erts_smp_atomic_t data;	/* Data associated with port. */
    Uint bytes_in;		/* Number of bytes read */
    Uint bytes_out;		/* Number of bytes written */

    ErlIOQueue ioq;              /* driver accessible i/o queue */
    DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
    char *name;		         /* String used in the open */
    erts_driver_t* drv_ptr;
    UWord drv_data;
    SWord os_pid;                /* Child process ID */
    ErtsProcList *suspended;	 /* List of suspended processes. */
    LineBuf *linebuf;            /* Buffer to hold data not ready for
				    process to get (line oriented I/O)*/
    erts_atomic32_t state;	 /* Status and type flags */
    int control_flags;		 /* Flags for port_control()  */
    ErlDrvPDL port_data_lock;

    ErtsPrtSD *psd;		 /* Port specific data */
    int reds; /* Only used while executing driver callbacks */
};


void erts_init_port_data(Port *);
void erts_cleanup_port_data(Port *);
Uint erts_port_data_size(Port *);
ErlOffHeap *erts_port_data_offheap(Port *);

#define ERTS_PORT_GET_CONNECTED(PRT) \
    ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
    erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
    erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))


struct erl_drv_port_data_lock {
    erts_mtx_t mtx;
    erts_atomic_t refc;
    Port *prt;
};

ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsRunQueue *
erts_port_runq(Port *prt)
{
#ifdef ERTS_SMP
    ErtsRunQueue *rq1, *rq2;
    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
    if (!rq1)
	return NULL;
    while (1) {
	erts_smp_runq_lock(rq1);
	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
	if (rq1 == rq2)
	    return rq1;
	erts_smp_runq_unlock(rq1);
	rq1 = rq2;
	if (!rq1)
	    return NULL;
    }
#else
# 235 "beam/erl_port.h"
    return ERTS_RUNQ_IX(0);
#endif
# 237 "beam/erl_port.h"
}

#endif
# 240 "beam/erl_port.h"


ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix);
ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_prtsd_get(Port *prt, int ix)
{
    return prt->psd ? prt->psd->data[ix] : NULL;
}

ERTS_GLB_INLINE void *
erts_prtsd_set(Port *prt, int ix, void *data)
{
    if (prt->psd) {
	void *old = prt->psd->data[ix];
	prt->psd->data[ix] = data;
	return old;
    }
    else {
	prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
	prt->psd->data[ix] = data;
	return NULL;
    }
}

#endif
# 269 "beam/erl_port.h"

Eterm erts_request_io_bytes(Process *c_p);


/* port status flags */

#define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
/* Port have begun exiting */
#define ERTS_PORT_SFLG_EXITING		((Uint32) (1 <<  1))
/* Distribution port */
#define ERTS_PORT_SFLG_DISTRIBUTION	((Uint32) (1 <<  2))
#define ERTS_PORT_SFLG_BINARY_IO	((Uint32) (1 <<  3))
#define ERTS_PORT_SFLG_SOFT_EOF		((Uint32) (1 <<  4))
/* Flow control */
/* Port is closing (no i/o accepted) */
#define ERTS_PORT_SFLG_CLOSING		((Uint32) (1 <<  5))
/* Send a closed message when terminating */
#define ERTS_PORT_SFLG_SEND_CLOSED	((Uint32) (1 <<  6))
/* Line orinted io on port */  
#define ERTS_PORT_SFLG_LINEBUF_IO	((Uint32) (1 <<  7))
/* Immortal port (only certain system ports) */
#define ERTS_PORT_SFLG_FREE		((Uint32) (1 <<  8))
#define ERTS_PORT_SFLG_INITIALIZING	((Uint32) (1 <<  9))
/* Port uses port specific locking (opposed to driver specific locking) */
#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 10))
#define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 11))
/* Last port to terminate halts the emulator */
#define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 12))
#ifdef DEBUG
/* Only debug: make sure all flags aren't cleared unintentionally */
#define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
#endif
# 301 "beam/erl_port.h"

/* Combinations of port status flags */ 
#define ERTS_PORT_SFLGS_DEAD						\
  (ERTS_PORT_SFLG_FREE | ERTS_PORT_SFLG_INITIALIZING)
#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
  (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID)
#define ERTS_PORT_SFLGS_INVALID_LOOKUP					\
  (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
   | ERTS_PORT_SFLG_EXITING						\
   | ERTS_PORT_SFLG_CLOSING)
#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP				\
  (ERTS_PORT_SFLGS_INVALID_LOOKUP					\
   | ERTS_PORT_SFLG_DISTRIBUTION)

/*
 * Costs in reductions for some port operations.
 */
#define ERTS_PORT_REDS_EXECUTE		(CONTEXT_REDS/4)
#define ERTS_PORT_REDS_FREE		(CONTEXT_REDS/400)
#define ERTS_PORT_REDS_TIMEOUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_INPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_OUTPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EVENT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUTV	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUT	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EXIT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CONNECT		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_UNLINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_LINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_BADSIG		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_CONTROL		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CALL		(CONTEXT_REDS/50)
#define ERTS_PORT_REDS_INFO		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_TERMINATE	(CONTEXT_REDS/50)

void print_port_info(Port *, int, void *);
void erts_port_free(Port *);
#ifndef ERTS_SMP
void erts_port_cleanup(Port *);
#endif
# 341 "beam/erl_port.h"
void erts_fire_port_monitor(Port *prt, Eterm ref);
#ifdef ERTS_SMP
int erts_port_handle_xports(Port *);
#endif
# 345 "beam/erl_port.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_is_port_locked(Port *);
#endif
# 349 "beam/erl_port.h"

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);

ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt)
{
    erts_ptab_atmc_inc_refc(&prt->common);
}

ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt)
{
    int referred = erts_ptab_atmc_dec_test_refc(&prt->common);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc)
{
    int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
{
    return erts_ptab_atmc_read_refc(&prt->common);
}

ERTS_GLB_INLINE int
erts_smp_port_trylock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    return erts_mtx_trylock(prt->lock);
#else
# 393 "beam/erl_port.h"
    return 0;
#endif
# 395 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_lock(prt->lock);
#endif
# 405 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_unlock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_unlock(prt->lock);
#endif
# 415 "beam/erl_port.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 418 "beam/erl_port.h"


#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS)			\
    (!(PP)							\
     || (erts_atomic32_read_nob(&(PP)->state) & (FLGS))	\
     || (PP)->common.id != (ID))

/* port lookup */

#define INVALID_PORT(PP, ID) \
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP)

/* Invalidate trace port if anything suspicious, for instance
 * that the port is a distribution port or it is busy.
 */
#define INVALID_TRACER_PORT(PP, ID)					\
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)

#define ERTS_PORT_SCHED_ID(P, ID) \
  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))

extern const Port erts_invalid_port;
#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)

int erts_is_port_ioq_empty(Port *);
void erts_terminate_port(Port *);

#ifdef ERTS_SMP
Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
#endif
# 448 "beam/erl_port.h"

ERTS_GLB_INLINE Port *erts_pix2port(int);
ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm);
ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32);
ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
ERTS_GLB_INLINE void erts_port_release(Port *);
#ifdef ERTS_SMP
ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
#endif
# 459 "beam/erl_port.h"
ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int);
ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *);
ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort);
ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm);
ERTS_GLB_INLINE int erts_is_port_alive(Eterm);
ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm);
ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *);

#define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL)

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Port *erts_pix2port(int ix)
{
    Port *prt;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_port));
    prt = (Port *) erts_ptab_pix2intptr_nob(&erts_port, ix);
    return prt == ERTS_PORT_LOCK_BUSY ? NULL : prt;
}

ERTS_GLB_INLINE Port *
erts_port_lookup_raw(Eterm id)
{
    Port *prt;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));
    return prt && prt->common.id == id ? prt : NULL;
}

ERTS_GLB_INLINE Port *
erts_port_lookup(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt = erts_port_lookup_raw(id);
    return (!prt
	    ? NULL
	    : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state))
	       ? NULL
	       : prt));
}


ERTS_GLB_INLINE Port*
erts_id2port(Eterm id)
{
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

    erts_smp_port_lock(prt);
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) {
	erts_smp_port_unlock(prt);
	return NULL;
    }

    return prt;
}


ERTS_GLB_INLINE Port*
erts_id2port_sflgs(Eterm id,
		   Process *c_p, ErtsProcLocks c_p_locks,
		   Uint32 invalid_sflgs)
{
#ifdef ERTS_SMP
    int no_proc_locks = !c_p || !c_p_locks;
#endif
# 543 "beam/erl_port.h"
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

#ifdef ERTS_SMP
    if (no_proc_locks)
	erts_smp_port_lock(prt);
    else if (erts_smp_port_trylock(prt) == EBUSY) {
	/* Unlock process locks, and acquire locks in lock order... */
	erts_smp_proc_unlock(c_p, c_p_locks);
	erts_smp_port_lock(prt);
	erts_smp_proc_lock(c_p, c_p_locks);
    }
#endif
# 568 "beam/erl_port.h"
    state = erts_atomic32_read_nob(&prt->state);
    if (state & invalid_sflgs) {
#ifdef ERTS_SMP
	erts_smp_port_unlock(prt);
#endif
# 573 "beam/erl_port.h"
	return NULL;
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_port_release(Port *prt)
{
    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
#ifdef ERTS_SMP
    erts_smp_port_unlock(prt);
#else
# 587 "beam/erl_port.h"
    if (prt->cleanup) {
	prt->cleanup = 0;
	erts_port_cleanup(prt);
    }
#endif
# 592 "beam/erl_port.h"
}

#ifdef ERTS_SMP

/*
 * erts_thr_id2port_sflgs() and erts_thr_port_release() can
 * be used by unmanaged threads in the SMP case.
 */
ERTS_GLB_INLINE Port *
erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt;
    ErtsThrPrgrDelayHandle dhndl;

    if (is_not_internal_port(id))
	return NULL;

    dhndl = erts_thr_progress_unmanaged_delay();

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id) {
	erts_thr_progress_unmanaged_continue(dhndl);
	prt = NULL;
    }
    else {
	erts_aint32_t state;
	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
	    erts_port_inc_refc(prt);
	    erts_thr_progress_unmanaged_continue(dhndl);
	}

	erts_mtx_lock(prt->lock);
	state = erts_atomic32_read_nob(&prt->state);
	if (state & invalid_sflgs) {
	    erts_mtx_unlock(prt->lock);
	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
		erts_port_dec_refc(prt);
	    prt = NULL;
	}
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_thr_port_release(Port *prt)
{
    erts_mtx_unlock(prt->lock);
#ifdef ERTS_SMP
    if (!erts_thr_progress_is_managed_thread())
	erts_port_dec_refc(prt);
#endif
# 646 "beam/erl_port.h"
}

#endif
# 649 "beam/erl_port.h"

ERTS_GLB_INLINE Port *
erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    ASSERT(prt != NULL);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;

    if (lock_pdl && prt->port_data_lock)
	driver_pdl_lock(prt->port_data_lock);

#if ERTS_ENABLE_LOCK_CHECK
    if (!ERTS_IS_CRASH_DUMPING) {
	if (erts_lc_is_emu_thr()) {
	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	    ERTS_LC_ASSERT(!prt->port_data_lock
			   || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
	else {
	    ERTS_LC_ASSERT(prt->port_data_lock);
	    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
    }
#endif
# 674 "beam/erl_port.h"

    if (erts_atomic32_read_nob(&prt->state)
	& ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
	if (lock_pdl && prt->port_data_lock)
	    driver_pdl_unlock(prt->port_data_lock);
	return ERTS_INVALID_ERL_DRV_PORT;
    }
    return prt;
}

ERTS_GLB_INLINE Port *
erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    erts_aint32_t state;
    ASSERT(prt);
    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;
    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
		       || ERTS_IS_CRASH_DUMPING);
    /* 
     * This state check is only needed since a driver callback
     * might terminate the port, and then call back into the
     * emulator. Drivers should preferably have been forbidden
     * to call into the emulator after terminating the port,
     * but it has been like this for ages. Perhaps forbid this
     * in some future major release?
     */
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
	return ERTS_INVALID_ERL_DRV_PORT;
    if (statep)
	*statep = state;
    return prt;
}

ERTS_GLB_INLINE Eterm
erts_drvport2id(ErlDrvPort drvport)
{
    Port *prt = erts_drvport2port(drvport);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return am_undefined;
    else
	return prt->common.id;
}

ERTS_GLB_INLINE Uint32
erts_portid2status(Eterm id)
{
    Port *prt = erts_port_lookup_raw(id);
    if (prt)
	return (Uint32) erts_atomic32_read_acqb(&prt->state);
    else
	return ERTS_PORT_SFLG_INVALID;
}

ERTS_GLB_INLINE int
erts_is_port_alive(Eterm id)
{
    return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID
				       | ERTS_PORT_SFLGS_DEAD));
}

ERTS_GLB_INLINE int
erts_is_valid_tracer_port(Eterm id)
{
    return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
}

ERTS_GLB_INLINE int
erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
{
    int reds = 0;
    erts_aint32_t state;

    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));

    state = erts_atomic32_read_nob(&prt->state);
    if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) {
	reds += ERTS_PORT_REDS_TERMINATE;
	erts_terminate_port(prt);
	state = erts_atomic32_read_nob(&prt->state);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
    }

#ifdef ERTS_SMP
    if (prt->xports) {
	reds += erts_port_handle_xports(prt);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	ASSERT(!prt->xports);
    }
#endif
# 767 "beam/erl_port.h"

    if (statep)
	*statep = state;

    return reds;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 775 "beam/erl_port.h"

void erts_port_resume_procs(Port *);

struct binary;

#define ERTS_P2P_SIG_TYPE_BAD			0
#define ERTS_P2P_SIG_TYPE_OUTPUT		1
#define ERTS_P2P_SIG_TYPE_OUTPUTV		2
#define ERTS_P2P_SIG_TYPE_CONNECT		3
#define ERTS_P2P_SIG_TYPE_EXIT			4
#define ERTS_P2P_SIG_TYPE_CONTROL		5
#define ERTS_P2P_SIG_TYPE_CALL			6
#define ERTS_P2P_SIG_TYPE_INFO			7
#define ERTS_P2P_SIG_TYPE_LINK			8
#define ERTS_P2P_SIG_TYPE_UNLINK		9

#define ERTS_P2P_SIG_TYPE_BITS			4
#define ERTS_P2P_SIG_TYPE_MASK \
    ((1 << ERTS_P2P_SIG_TYPE_BITS) - 1)

#define ERTS_P2P_SIG_DATA_FLG(N) \
    (1 << (ERTS_P2P_SIG_TYPE_BITS + (N)))
#define ERTS_P2P_SIG_DATA_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG(0)
#define ERTS_P2P_SIG_DATA_FLG_REPLY		ERTS_P2P_SIG_DATA_FLG(1)
#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG(2)
#define ERTS_P2P_SIG_DATA_FLG_FORCE		ERTS_P2P_SIG_DATA_FLG(3)
#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT	ERTS_P2P_SIG_DATA_FLG(4)
#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK	ERTS_P2P_SIG_DATA_FLG(5)
#define ERTS_P2P_SIG_DATA_FLG_SCHED		ERTS_P2P_SIG_DATA_FLG(6)
#define ERTS_P2P_SIG_DATA_FLG_ASYNC		ERTS_P2P_SIG_DATA_FLG(7)

struct ErtsProc2PortSigData_ {
    int flags;
    Eterm caller;
    Uint32 ref[ERTS_MAX_REF_NUMBERS];
    union {
	struct {
	    Eterm from;
	    ErlIOVec *evp;
	    ErlDrvBinary *cbinp;
	} outputv;
	struct {
	    Eterm from;
	    char *bufp;
	    ErlDrvSizeT size;
	} output;
	struct {
	    Eterm from;
	    Eterm connected;
	} connect;
	struct {
	    Eterm from;
	    Eterm reason;
	    ErlHeapFragment *bp;
	} exit;
	struct {
	    struct binary *binp;
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} control;
	struct {
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} call;
	struct {
	    Eterm item;
	} info;
	struct {
	    Eterm port;
	    Eterm to;
	} link;
	struct {
	    Eterm from;
	} unlink;
    } u;
} ;

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp);
ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return !0;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return !0;
    default:				return 0;
    }
}

ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return sigdp->u.output.size;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return sigdp->u.outputv.evp->size;
    default:				return (ErlDrvSizeT) 0;
    }
}

#endif
# 882 "beam/erl_port.h"

#define ERTS_PROC2PORT_SIG_EXEC			0
#define ERTS_PROC2PORT_SIG_ABORT		1
#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND	2
#define ERTS_PROC2PORT_SIG_ABORT_CLOSED		3

typedef int (*ErtsProc2PortSigCallback)(Port *,
					erts_aint32_t,
					int,
					ErtsProc2PortSigData *);

typedef enum {
    ERTS_PORT_OP_BADARG,
    ERTS_PORT_OP_CALLER_EXIT,
    ERTS_PORT_OP_BUSY,
    ERTS_PORT_OP_BUSY_SCHEDULED,
    ERTS_PORT_OP_SCHEDULED,
    ERTS_PORT_OP_DROPPED,
    ERTS_PORT_OP_DONE
} ErtsPortOpResult;

ErtsPortOpResult
erts_schedule_proc2port_signal(Process *,
			       Port *,
			       Eterm,
			       Eterm *,
			       ErtsProc2PortSigData *,
			       int,
			       ErtsPortTaskHandle *,
			       ErtsProc2PortSigCallback);

int erts_deliver_port_exit(Port *, Eterm, Eterm, int);

/*
 * Port signal flags
 */
#define ERTS_PORT_SIG_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG_BANG_OP
#define ERTS_PORT_SIG_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG_NOSUSPEND
#define ERTS_PORT_SIG_FLG_FORCE			ERTS_P2P_SIG_DATA_FLG_FORCE
#define ERTS_PORT_SIG_FLG_BROKEN_LINK		ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK
#define ERTS_PORT_SIG_FLG_BAD_OUTPUT		ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT
#define ERTS_PORT_SIG_FLG_FORCE_SCHED		ERTS_P2P_SIG_DATA_FLG_SCHED
#define ERTS_PORT_SIG_FLG_ASYNC			ERTS_P2P_SIG_DATA_FLG_ASYNC
/* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */
#define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL	ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT

/*
 * Port ! {Owner, {command, Data}}
 * Port ! {Owner, {connect, NewOwner}}
 * Port ! {Owner, close}
 */
ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);

/*
 * Signals from processes to ports.
 */
ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);

#endif
# 948 "beam/erl_port.h"
# 31 "beam/erl_hl_timer.h" 2
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_hl_timer.h"
# 33 "beam/erl_hl_timer.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc_types.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_hl_timer.h"
# 34 "beam/erl_hl_timer.h"

#define ERTS_PTMR_NONE ((erts_aint_t) NULL)
#define ERTS_PTMR_TIMEDOUT (ERTS_PTMR_NONE + ((erts_aint_t) 1))

#define ERTS_PTMR_INIT(P) \
    erts_smp_atomic_init_nob(&(P)->common.timer, ERTS_PTMR_NONE)
#define ERTS_PTMR_IS_SET(P) \
    (ERTS_PTMR_NONE != erts_smp_atomic_read_nob(&(P)->common.timer))
#define ERTS_PTMR_IS_TIMED_OUT(P) \
    (ERTS_PTMR_TIMEDOUT == erts_smp_atomic_read_nob(&(P)->common.timer))

#define ERTS_PTMR_CLEAR(P)					\
    do {							\
	ASSERT(ERTS_PTMR_IS_TIMED_OUT((P)));			\
	erts_smp_atomic_set_nob(&(P)->common.timer,		\
				ERTS_PTMR_NONE);		\
    } while (0)

size_t erts_timer_type_size(ErtsAlcType_t type);
int erts_set_proc_timer_term(Process *, Eterm);
void erts_set_proc_timer_uword(Process *, UWord);
void erts_cancel_proc_timer(Process *);
void erts_set_port_timer(Port *, Sint64);
void erts_cancel_port_timer(Port *);
Sint64 erts_read_port_timer(Port *);
int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **);
int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **);
ErtsHLTimerService *erts_create_timer_service(void);
void erts_hl_timer_init(void);
void erts_start_timer_callback(ErtsMonotonicTime,
			       void (*)(void *),
			       void *);
#ifdef ERTS_SMP
void
erts_handle_canceled_timers(void *vesdp,
			    int *need_thr_progress,
			    ErtsThrPrgrVal *thr_prgr_p,
			    int *need_more_work);
#endif
# 73 "beam/erl_hl_timer.h"

Uint erts_bif_timer_memory_size(void);
void erts_print_bif_timer_info(int to, void *to_arg);

void erts_debug_bif_timer_foreach(void (*func)(Eterm,
					       Eterm,
					       ErlHeapFragment *,
					       void *),
				  void *arg);
void
erts_debug_callback_timer_foreach(void (*tclbk)(void *),
				  void (*func)(void *,
					       ErtsMonotonicTime,
					       void *),
				  void *arg);
#endif /* ERL_HL_TIMER_H__ */
# 89 "beam/erl_hl_timer.h"
# 57 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 1 "beam/erl_time.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2006-2011. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef ERL_TIME_H__
#define ERL_TIME_H__

/* timer wheel size NEED to be a power of 2 */
#ifdef SMALL_MEMORY
#define ERTS_TIW_SIZE (1 << 13)
#else
# 28 "beam/erl_time.h"
#define ERTS_TIW_SIZE (1 << 16)
#endif
# 30 "beam/erl_time.h"

#if defined(DEBUG) || 0
#define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B)
#else
# 34 "beam/erl_time.h"
#define ERTS_TIME_ASSERT(B) ((void) 1)
#endif
# 36 "beam/erl_time.h"

typedef enum {
    ERTS_NO_TIME_WARP_MODE,
    ERTS_SINGLE_TIME_WARP_MODE,
    ERTS_MULTI_TIME_WARP_MODE
} ErtsTimeWarpMode;

typedef struct ErtsTimerWheel_ ErtsTimerWheel;
typedef ErtsMonotonicTime * ErtsNextTimeoutRef;

extern SysTimeval erts_first_emu_time;


void erts_monitor_time_offset(Eterm id, Eterm ref);
int erts_demonitor_time_offset(Eterm ref);

int erts_init_time_sup(int, ErtsTimeWarpMode);
void erts_late_init_time_sup(void);

ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *);
void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode);
void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime);
Uint erts_timer_wheel_memory_size(void);

#ifdef DEBUG
void erts_p_slpq(void);
#endif
# 63 "beam/erl_time.h"

/* time_sup */

#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME))
#  ifndef HAVE_ERTS_NOW_CPU
#    define HAVE_ERTS_NOW_CPU
#    ifdef HAVE_GETHRVTIME
#      define erts_start_now_cpu() sys_start_hrvtime()
#      define erts_stop_now_cpu()  sys_stop_hrvtime()
#    endif
# 73 "beam/erl_time.h"
#  endif
# 74 "beam/erl_time.h"
void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec);
#endif
# 76 "beam/erl_time.h"

typedef UWord erts_approx_time_t;
erts_approx_time_t erts_get_approx_time(void);

int erts_has_time_correction(void);
int erts_check_time_adj_support(int time_correction,
				ErtsTimeWarpMode time_warp_mode);

ErtsTimeWarpMode erts_time_warp_mode(void);

typedef enum {
    ERTS_TIME_OFFSET_PRELIMINARY,
    ERTS_TIME_OFFSET_FINAL,
    ERTS_TIME_OFFSET_VOLATILE
} ErtsTimeOffsetState;

ErtsTimeOffsetState erts_time_offset_state(void); 
ErtsTimeOffsetState erts_finalize_time_offset(void); 
struct process;
Eterm erts_get_monotonic_start_time(struct process *c_p);
Eterm erts_get_monotonic_end_time(struct process *c_p);
Eterm erts_monotonic_time_source(struct process*c_p);
Eterm erts_system_time_source(struct process*c_p);

#ifdef SYS_CLOCK_RESOLUTION
#define ERTS_CLKTCK_RESOLUTION ((ErtsMonotonicTime) (SYS_CLOCK_RESOLUTION*1000))
#else
# 103 "beam/erl_time.h"
#define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution)
#endif
# 105 "beam/erl_time.h"

#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000))

struct erts_time_sup_read_only__ {
    ErtsMonotonicTime monotonic_time_unit;
#if !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT
    ErtsMonotonicTime start;
    struct {
	ErtsMonotonicTime native;
	ErtsMonotonicTime nsec;
	ErtsMonotonicTime usec;
	ErtsMonotonicTime msec;
	ErtsMonotonicTime sec;
    } start_offset;
#endif
# 120 "beam/erl_time.h"
#ifndef SYS_CLOCK_RESOLUTION
    ErtsMonotonicTime clktck_resolution;
#endif
# 123 "beam/erl_time.h"
};

typedef struct {
    union {
	struct erts_time_sup_read_only__ o;
	char align__[(((sizeof(struct erts_time_sup_read_only__) - 1)
		       / ASSUMED_CACHE_LINE_SIZE) + 1)
		     * ASSUMED_CACHE_LINE_SIZE];
    } r;
} ErtsTimeSupData;

extern ErtsTimeSupData erts_time_sup__;

ErtsMonotonicTime erts_napi_monotonic_time(int time_unit);
ErtsMonotonicTime erts_napi_time_offset(int time_unit);
ErtsMonotonicTime erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to);

ERTS_GLB_INLINE Uint64
erts_time_unit_conversion(Uint64 value,
			  Uint32 from_time_unit,
			  Uint32 to_time_unit);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Uint64
erts_time_unit_conversion(Uint64 value,
			  Uint32 from_time_unit,
			  Uint32 to_time_unit)
{
    Uint64 high, low, result;
    if (value <= ~((Uint64) 0)/to_time_unit)
	return (value*to_time_unit)/from_time_unit;

    low = value & ((Uint64) 0xffffffff);
    high = (value >> 32) & ((Uint64) 0xffffffff);

    low *= to_time_unit;
    high *= to_time_unit;

    high += (low >> 32) & ((Uint64) 0xffffffff);
    low &= ((Uint64) 0xffffffff);

    result = high % from_time_unit;
    high /= from_time_unit;
    high <<= 32;

    result <<= 32;
    result += low;
    result /= from_time_unit;
    result += high;

    return result;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 178 "beam/erl_time.h"

/*
 * Range of monotonic time internally
 */

#define ERTS_MONOTONIC_BEGIN						\
    ERTS_MONOTONIC_TIME_UNIT
#define ERTS_MONOTONIC_END						\
    ((ERTS_MONOTONIC_TIME_MAX / ERTS_MONOTONIC_TIME_UNIT)		\
     * ERTS_MONOTONIC_TIME_UNIT)

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT

/*
 * If the monotonic time unit is a compile time constant,
 * it is assumed (and need) to be a power of 10.
 */

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT < 1000*1000
#  error Compile time time unit needs to be at least 1000000
#endif
# 199 "beam/erl_time.h"

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000*1000
/* Nano-second time unit */

#define ERTS_MONOTONIC_TO_SEC__(NSEC) ((NSEC) / (1000*1000*1000))
#define ERTS_MONOTONIC_TO_MSEC__(NSEC) ((NSEC) / (1000*1000))
#define ERTS_MONOTONIC_TO_USEC__(NSEC) ((NSEC) / 1000)
#define ERTS_MONOTONIC_TO_NSEC__(NSEC) (NSEC)

#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000*1000))
#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*(1000*1000))
#define ERTS_USEC_TO_MONOTONIC__(USEC) (((ErtsMonotonicTime) (USEC))*1000)
#define ERTS_NSEC_TO_MONOTONIC__(NSEC) ((ErtsMonotonicTime) (NSEC))

#elif ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT == 1000*1000
# 214 "beam/erl_time.h"
/* Micro-second time unit */

#define ERTS_MONOTONIC_TO_SEC__(USEC) ((USEC) / (1000*1000))
#define ERTS_MONOTONIC_TO_MSEC__(USEC) ((USEC) / 1000)
#define ERTS_MONOTONIC_TO_USEC__(USEC) (USEC)
#define ERTS_MONOTONIC_TO_NSEC__(USEC) ((USEC)*1000)

#define ERTS_SEC_TO_MONOTONIC__(SEC) (((ErtsMonotonicTime) (SEC))*(1000*1000))
#define ERTS_MSEC_TO_MONOTONIC__(MSEC) (((ErtsMonotonicTime) (MSEC))*1000)
#define ERTS_USEC_TO_MONOTONIC__(USEC) ((ErtsMonotonicTime) (USEC))
#define ERTS_NSEC_TO_MONOTONIC__(NSEC) (((ErtsMonotonicTime) (NSEC))/1000)

#else
# 227 "beam/erl_time.h"
#error Missing implementation for monotonic time unit
#endif
# 229 "beam/erl_time.h"

#define ERTS_MONOTONIC_TIME_UNIT \
    ((ErtsMonotonicTime) ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT)

/*
 * NOTE! ERTS_MONOTONIC_TIME_START_EXTERNAL *need* to be a multiple
 *       of ERTS_MONOTONIC_TIME_UNIT.
 */

#ifdef ARCH_32
/*
 * Want to use a big-num of arity 2 as long as possible (584 years
 * in the nano-second time unit case).
 */
#define ERTS_MONOTONIC_TIME_START_EXTERNAL		\
    (((((((ErtsMonotonicTime) 1) << 32)-1)	\
       / ERTS_MONOTONIC_TIME_UNIT)		\
      * ERTS_MONOTONIC_TIME_UNIT)		\
     + ERTS_MONOTONIC_TIME_UNIT)

#else /* ARCH_64 */
# 250 "beam/erl_time.h"

#if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT <= 10*1000*1000

/*
 * Using micro second time unit or lower. Start at zero since
 * time will remain an immediate for a very long time anyway
 * (1827 years in the 10 micro second case)...
 */
#define ERTS_MONOTONIC_TIME_START_EXTERNAL ((ErtsMonotonicTime) 0)

#else /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 10*1000*1000 */
# 261 "beam/erl_time.h"

/*
 * Want to use an immediate as long as possible (36 years in the
 * nano-second time unit case).
*/
#define ERTS_MONOTONIC_TIME_START_EXTERNAL 		\
    ((((ErtsMonotonicTime) MIN_SMALL)		\
      / ERTS_MONOTONIC_TIME_UNIT)		\
     * ERTS_MONOTONIC_TIME_UNIT)

#endif /* ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT > 1000*1000 */
# 272 "beam/erl_time.h"

#endif /* ARCH_64 */
# 274 "beam/erl_time.h"

/*
 * Offsets from internal monotonic time to external monotonic time
 */

#define ERTS_MONOTONIC_OFFSET_NATIVE \
    (ERTS_MONOTONIC_TIME_START_EXTERNAL - ERTS_MONOTONIC_BEGIN)
#define ERTS_MONOTONIC_OFFSET_NSEC					\
    ERTS_MONOTONIC_TO_NSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
#define ERTS_MONOTONIC_OFFSET_USEC					\
    ERTS_MONOTONIC_TO_USEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
#define ERTS_MONOTONIC_OFFSET_MSEC					\
    ERTS_MONOTONIC_TO_MSEC__(ERTS_MONOTONIC_OFFSET_NATIVE)
#define ERTS_MONOTONIC_OFFSET_SEC					\
    ERTS_MONOTONIC_TO_SEC__(ERTS_MONOTONIC_OFFSET_NATIVE)

#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
    ((MON) / (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))
#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
    ((TCKS) * (ERTS_MONOTONIC_TIME_UNIT/ERTS_CLKTCK_RESOLUTION))

#else /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
# 296 "beam/erl_time.h"

/*
 * Initialized in erts_init_sys_time_sup()
 */
#define ERTS_MONOTONIC_TIME_UNIT (erts_time_sup__.r.o.monotonic_time_unit)

/*
 * Offsets from internal monotonic time to external monotonic time
 *
 * Initialized in erts_init_time_sup()...
 */
#define ERTS_MONOTONIC_TIME_START_EXTERNAL (erts_time_sup__.r.o.start)
#define ERTS_MONOTONIC_OFFSET_NATIVE (erts_time_sup__.r.o.start_offset.native)
#define ERTS_MONOTONIC_OFFSET_NSEC (erts_time_sup__.r.o.start_offset.nsec)
#define ERTS_MONOTONIC_OFFSET_USEC (erts_time_sup__.r.o.start_offset.usec)
#define ERTS_MONOTONIC_OFFSET_MSEC (erts_time_sup__.r.o.start_offset.msec)
#define ERTS_MONOTONIC_OFFSET_SEC (erts_time_sup__.r.o.start_offset.sec)

#define ERTS_CONV_FROM_MON_UNIT___(M, TO)				\
    ((ErtsMonotonicTime)						\
     erts_time_unit_conversion((Uint64) (M),				\
			       (Uint32) ERTS_MONOTONIC_TIME_UNIT,	\
			       (Uint32) (TO)))

#define ERTS_CONV_TO_MON_UNIT___(M, FROM)				\
    ((ErtsMonotonicTime)						\
     erts_time_unit_conversion((Uint64) (M),				\
			       (Uint32) (FROM),				\
			       (Uint32) ERTS_MONOTONIC_TIME_UNIT))	\

#define ERTS_MONOTONIC_TO_SEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1)
#define ERTS_MONOTONIC_TO_MSEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1000)
#define ERTS_MONOTONIC_TO_USEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000)
#define ERTS_MONOTONIC_TO_NSEC__(M) \
    ERTS_CONV_FROM_MON_UNIT___((M), 1000*1000*1000)

#define ERTS_SEC_TO_MONOTONIC__(SEC) \
    ERTS_CONV_TO_MON_UNIT___((SEC), 1)
#define ERTS_MSEC_TO_MONOTONIC__(MSEC) \
    ERTS_CONV_TO_MON_UNIT___((MSEC), 1000)
#define ERTS_USEC_TO_MONOTONIC__(USEC) \
    ERTS_CONV_TO_MON_UNIT___((USEC), 1000*1000)
#define ERTS_NSEC_TO_MONOTONIC__(NSEC) \
    ERTS_CONV_TO_MON_UNIT___((NSEC), 1000*1000*1000)

#define ERTS_MONOTONIC_TO_CLKTCKS__(MON) \
    ERTS_CONV_FROM_MON_UNIT___((MON), ERTS_CLKTCK_RESOLUTION)
#define ERTS_CLKTCKS_TO_MONOTONIC__(TCKS) \
    ERTS_CONV_TO_MON_UNIT___((TCKS), ERTS_CLKTCK_RESOLUTION)

#endif /* !ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT */
# 350 "beam/erl_time.h"

#define ERTS_MONOTONIC_TIME_END_EXTERNAL				\
    (ERTS_MONOTONIC_TIME_START_EXTERNAL < 0				\
     ? (ERTS_MONOTONIC_TIME_START_EXTERNAL				\
	+ (ERTS_MONOTONIC_END - ERTS_MONOTONIC_BEGIN))			\
     : (ERTS_MONOTONIC_END - ERTS_MONOTONIC_TIME_START_EXTERNAL))

#define ERTS_MSEC_TO_CLKTCKS__(MON) \
    ((MON) * (ERTS_CLKTCK_RESOLUTION/1000))
#define ERTS_CLKTCKS_TO_MSEC__(TCKS) \
    ((TCKS) / (ERTS_CLKTCK_RESOLUTION/1000))

#define ERTS_MONOTONIC_TO_SEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_SEC__((X)))
#define ERTS_MONOTONIC_TO_MSEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_MSEC__((X)))
#define ERTS_MONOTONIC_TO_USEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_USEC__((X)))
#define ERTS_MONOTONIC_TO_NSEC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MONOTONIC_TO_NSEC__((X)))
#define ERTS_SEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_SEC_TO_MONOTONIC__((X)))
#define ERTS_MSEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_MSEC_TO_MONOTONIC__((X)))
#define ERTS_USEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_USEC_TO_MONOTONIC__((X)))
#define ERTS_NSEC_TO_MONOTONIC(X)	\
    (ERTS_TIME_ASSERT((X) >= 0),	\
     ERTS_NSEC_TO_MONOTONIC__((X)))

#define ERTS_MONOTONIC_TO_CLKTCKS(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_MONOTONIC_TO_CLKTCKS__((X)))
#define ERTS_CLKTCKS_TO_MONOTONIC(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_CLKTCKS_TO_MONOTONIC__((X)))

#define ERTS_MSEC_TO_CLKTCKS(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_MSEC_TO_CLKTCKS__((X)))
#define ERTS_CLKTCKS_TO_MSEC(X) \
    (ERTS_TIME_ASSERT((X) >= 0),		\
     ERTS_CLKTCKS_TO_MSEC__((X)))

#endif /* ERL_TIME_H__ */
# 402 "beam/erl_time.h"

/* timer-wheel api */
#if defined(ERTS_WANT_TIMER_WHEEL_API) && !defined(ERTS_GOT_TIMER_WHEEL_API)
#define ERTS_GOT_TIMER_WHEEL_API

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 407 "beam/erl_time.h"
# 408 "beam/erl_time.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 408 "beam/erl_time.h"
# 409 "beam/erl_time.h"

void erts_sched_init_time_sup(ErtsSchedulerData *esdp);


#define ERTS_TWHEEL_SLOT_AT_ONCE -1
#define ERTS_TWHEEL_SLOT_INACTIVE -2

/*
** Timer entry:
*/
typedef struct erl_timer {
    struct erl_timer* next;	/* next entry tiw slot or chain */
    struct erl_timer* prev;	/* prev entry tiw slot or chain */
    union {
	struct {
	    void (*timeout)(void*); /* called when timeout */
	    void (*cancel)(void*);  /* called when cancel (may be NULL) */
	    void* arg;              /* argument to timeout/cancel procs */
	} func;
	ErtsThrPrgrLaterOp cleanup;
    } u;
    ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
    int slot;
} ErtsTWheelTimer;

typedef void (*ErlTimeoutProc)(void*);
typedef void (*ErlCancelProc)(void*);

void erts_twheel_set_timer(ErtsTimerWheel *tiw,
			   ErtsTWheelTimer *p, ErlTimeoutProc timeout,
			   ErlCancelProc cancel, void *arg,
			   ErtsMonotonicTime timeout_pos);
void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p);
ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp);

ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *);

ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p);
ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p)
{
    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
}

ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
{
    return *((ErtsMonotonicTime *) nxt_tmo_ref);
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 462 "beam/erl_time.h"

void
erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
			  void (*tclbk)(void *),
			  void (*func)(void *,
				       ErtsMonotonicTime,
				       void *),
			  void *arg);

#endif /* timer wheel api */
# 472 "beam/erl_time.h"
# 58 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 1 "beam/external.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Same order as the ordering of terms in erlang */

/* Since there are 255 different External tag values to choose from
   There is no reason to not be extravagant.
   Hence, the different tags for large/small tuple e.t.c
*/


#ifdef ERTS_WANT_EXTERNAL_TAGS
#ifndef ERTS_EXTERNAL_TAGS
#define ERTS_EXTERNAL_TAGS

#define SMALL_INTEGER_EXT 'a'
#define INTEGER_EXT       'b'
#define FLOAT_EXT         'c'
#define ATOM_EXT          'd'
#define SMALL_ATOM_EXT    's'
#define REFERENCE_EXT     'e'
#define NEW_REFERENCE_EXT 'r'
#define PORT_EXT          'f'
#define NEW_FLOAT_EXT     'F'
#define PID_EXT           'g'
#define SMALL_TUPLE_EXT   'h'
#define LARGE_TUPLE_EXT   'i'
#define NIL_EXT           'j'
#define STRING_EXT        'k'
#define LIST_EXT          'l'
#define BINARY_EXT        'm'
#define BIT_BINARY_EXT    'M'
#define SMALL_BIG_EXT     'n'
#define LARGE_BIG_EXT     'o'
#define NEW_FUN_EXT       'p'
#define EXPORT_EXT        'q'
#define MAP_EXT           't'
#define FUN_EXT           'u'
#define ATOM_UTF8_EXT     'v'
#define SMALL_ATOM_UTF8_EXT 'w'

#define DIST_HEADER       'D'
#define ATOM_CACHE_REF    'R'
#define ATOM_INTERNAL_REF2 'I'
#define ATOM_INTERNAL_REF3 'K'
#define BINARY_INTERNAL_REF 'J'
#define BIT_BINARY_INTERNAL_REF 'L'
#define COMPRESSED        'P'

#if 0
/* Not used anymore */
#define CACHED_ATOM       'C'
#define NEW_CACHE         'N'
#endif
# 72 "beam/external.h"


#define VERSION_MAGIC 131   /* 130 in erlang 4.2 */
                /* Increment this when changing the external format. */
                /* ON the other hand, don't change the external format */
                /* since that breaks other people's code! */

#endif /* ERTS_EXTERNAL_TAGS */
# 80 "beam/external.h"
#endif /* ERTS_WANT_EXTERNAL_TAGS */
# 81 "beam/external.h"

#ifndef ERL_EXTERNAL_H__
#define ERL_EXTERNAL_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 85 "beam/external.h"
# 86 "beam/external.h"

#define ERTS_ATOM_CACHE_SIZE 2048

typedef struct cache {
    Eterm in_arr[ERTS_ATOM_CACHE_SIZE];
    Eterm out_arr[ERTS_ATOM_CACHE_SIZE];
} ErtsAtomCache;

typedef struct {
    int hdr_sz;
    int sz;
    int long_atoms;
    int cix[ERTS_ATOM_CACHE_SIZE];
    struct {
	Eterm atom;
	int iix;
    } cache[ERTS_ATOM_CACHE_SIZE];
} ErtsAtomCacheMap;

typedef struct {
    Uint32 size;
    Eterm atom[ERTS_ATOM_CACHE_SIZE];
} ErtsAtomTranslationTable;

/*
 * These flags are tagged onto the high bits of a connection ID and stored in
 * the ErtsDistExternal structure's flags field.  They are used to indicate
 * various bits of state necessary to decode binaries in a variety of
 * scenarios. The mask ERTS_DIST_EXT_CON_ID_MASK is used later to separate the
 * connection ID from the flags. Be careful to ensure that the mask does not
 * overlap any of the bits used for flags, or ERTS will leak flags bits into
 * connection IDs and leak connection ID bits into the flags.
 */
#define ERTS_DIST_EXT_DFLAG_HDR      ((Uint32) 0x80000000)
#define ERTS_DIST_EXT_ATOM_TRANS_TAB ((Uint32) 0x40000000)
#define ERTS_DIST_EXT_BTT_SAFE       ((Uint32) 0x20000000)
#define ERTS_DIST_EXT_CON_ID_MASK    ((Uint32) 0x1fffffff)

#define ERTS_DIST_EXT_CON_ID(DIST_EXTP) \
  ((DIST_EXTP)->flags & ERTS_DIST_EXT_CON_ID_MASK)
typedef struct {
    DistEntry *dep;
    byte *extp;
    byte *ext_endp;
    Sint heap_size;
    Uint32 flags;
    ErtsAtomTranslationTable attab;
} ErtsDistExternal;

typedef struct {
    int have_header;
    int cache_entries;
} ErtsDistHeaderPeek;

#define ERTS_DIST_EXT_SIZE(EDEP) \
  (sizeof(ErtsDistExternal) \
   - (((EDEP)->flags & ERTS_DIST_EXT_ATOM_TRANS_TAB) \
      ? (ASSERT(0 <= (EDEP)->attab.size \
		&& (EDEP)->attab.size <= ERTS_ATOM_CACHE_SIZE), \
	 sizeof(Eterm)*(ERTS_ATOM_CACHE_SIZE - (EDEP)->attab.size)) \
      : sizeof(ErtsAtomTranslationTable)))

typedef struct {
    byte *extp;
    int exttmp;
    Uint extsize;
    Uint heap_size;
} ErtsBinary2TermState;


/* -------------------------------------------------------------------------- */

void erts_init_atom_cache_map(ErtsAtomCacheMap *);
void erts_reset_atom_cache_map(ErtsAtomCacheMap *);
void erts_destroy_atom_cache_map(ErtsAtomCacheMap *);
void erts_finalize_atom_cache_map(ErtsAtomCacheMap *, Uint32);

Uint erts_encode_ext_dist_header_size(ErtsAtomCacheMap *);
byte *erts_encode_ext_dist_header_setup(byte *, ErtsAtomCacheMap *);
byte *erts_encode_ext_dist_header_finalize(byte *, ErtsAtomCache *, Uint32);
struct erts_dsig_send_context;
int erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap*, Uint* szp);
int erts_encode_dist_ext_size_int(Eterm term, struct erts_dsig_send_context* ctx, Uint* szp);
struct TTBEncodeContext_;
int erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *,
			  struct TTBEncodeContext_ *, Sint* reds);

Uint erts_encode_ext_size(Eterm);
Uint erts_encode_ext_size_2(Eterm, unsigned);
Uint erts_encode_ext_size_ets(Eterm);
void erts_encode_ext(Eterm, byte **);
byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap);

#ifdef ERTS_WANT_EXTERNAL_TAGS
ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint);
#endif
# 182 "beam/external.h"
ERTS_GLB_INLINE void erts_free_dist_ext_copy(ErtsDistExternal *);
ERTS_GLB_INLINE void *erts_dist_ext_trailer(ErtsDistExternal *);
ErtsDistExternal *erts_make_dist_ext_copy(ErtsDistExternal *, Uint);
void *erts_dist_ext_trailer(ErtsDistExternal *);
void erts_destroy_dist_ext_copy(ErtsDistExternal *);
int erts_prepare_dist_ext(ErtsDistExternal *, byte *, Uint,
			  DistEntry *, ErtsAtomCache *);
Sint erts_decode_dist_ext_size(ErtsDistExternal *);
Eterm erts_decode_dist_ext(ErtsHeapFactory* factory, ErtsDistExternal *);

Sint erts_decode_ext_size(byte*, Uint);
Sint erts_decode_ext_size_ets(byte*, Uint);
Eterm erts_decode_ext(ErtsHeapFactory*, byte**);
Eterm erts_decode_ext_ets(ErtsHeapFactory*, byte*);

Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags);

Sint erts_binary2term_prepare(ErtsBinary2TermState *, byte *, Sint);
void erts_binary2term_abort(ErtsBinary2TermState *);
Eterm erts_binary2term_create(ErtsBinary2TermState *, ErtsHeapFactory*);
int erts_debug_max_atom_out_cache_index(void);
int erts_debug_atom_to_out_cache_index(Eterm);


#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#ifdef ERTS_WANT_EXTERNAL_TAGS
ERTS_GLB_INLINE void
erts_peek_dist_header(ErtsDistHeaderPeek *dhpp, byte *ext, Uint sz)
{
    if (ext[0] == VERSION_MAGIC
	|| ext[1] != DIST_HEADER
	|| sz < (1+1+1))
	dhpp->have_header = 0;
    else {
	dhpp->have_header = 1;
	dhpp->cache_entries = (int) get_int8(&ext[2]);
    }
}
#endif
# 221 "beam/external.h"

ERTS_GLB_INLINE void
erts_free_dist_ext_copy(ErtsDistExternal *edep)
{
    if (edep->dep)
	erts_deref_dist_entry(edep->dep);
    erts_free(ERTS_ALC_T_EXT_TERM_DATA, edep);
}

ERTS_GLB_INLINE void *
erts_dist_ext_trailer(ErtsDistExternal *edep)
{
    void *res = (void *) (edep->ext_endp
			  + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp));
    ASSERT((((UWord) res) % sizeof(Uint)) == 0);
    return res;
}

#endif
# 240 "beam/external.h"

#endif /* ERL_EXTERNAL_H__ */
# 242 "beam/external.h"
# 60 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 1 "sys/common/erl_mseg.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2002-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_MSEG_H_
#define ERL_MSEG_H_

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 24 "sys/common/erl_mseg.h"
# 25 "sys/common/erl_mseg.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc_types.h"
#endif /* expanded by -frewrite-includes */
# 25 "sys/common/erl_mseg.h"
# 26 "sys/common/erl_mseg.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mmap.h"
#endif /* expanded by -frewrite-includes */
# 26 "sys/common/erl_mseg.h"
# 1 "sys/common/erl_mmap.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_MMAP_H__
#define ERL_MMAP_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 24 "sys/common/erl_mmap.h"
# 25 "sys/common/erl_mmap.h"

#define ERTS_MMAP_SUPERALIGNED_BITS (18)
/* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */

#define ERTS_MMAPFLG_OS_ONLY			(((Uint32) 1) << 0)
#define ERTS_MMAPFLG_SUPERCARRIER_ONLY		(((Uint32) 1) << 1)
#define ERTS_MMAPFLG_SUPERALIGNED		(((Uint32) 1) << 2)

#define ERTS_HAVE_ERTS_OS_MMAP			(1 << 0)
#define ERTS_HAVE_ERTS_SUPERCARRIER_MMAP	(1 << 1)
extern int erts_have_erts_mmap;
extern UWord erts_page_inv_mask;

typedef struct {
    struct {
	char *start;
	char *end;
    } virtual_range;
    struct {
	char *start;
	char *end;
    } predefined_area;
    UWord scs;  /* super carrier size */
    int sco;    /* super carrier only? */
    UWord scrfsd; /* super carrier reserved free segment descriptors */
    int scrpm; /* super carrier reserve physical memory */
}ErtsMMapInit;

#define ERTS_MMAP_INIT_DEFAULT_INITER \
    {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1}

void *erts_mmap(Uint32 flags, UWord *sizep);
void erts_munmap(Uint32 flags, void *ptr, UWord size);
void *erts_mremap(Uint32 flags, void *ptr, UWord old_size, UWord *sizep);
int erts_mmap_in_supercarrier(void *ptr);
void erts_mmap_init(ErtsMMapInit*);
struct erts_mmap_info_struct
{
    UWord sizes[6];
    UWord segs[6];
    UWord os_used;
};
Eterm erts_mmap_info(int *print_to_p, void *print_to_arg,
                     Eterm** hpp, Uint* szp, struct erts_mmap_info_struct*);
Eterm erts_mmap_info_options(char *prefix, int *print_to_p, void *print_to_arg,
                             Uint **hpp, Uint *szp);
struct process;
Eterm erts_mmap_debug_info(struct process*);

#define ERTS_SUPERALIGNED_SIZE \
    (1 << ERTS_MMAP_SUPERALIGNED_BITS)
#define ERTS_INV_SUPERALIGNED_MASK \
    ((UWord) (ERTS_SUPERALIGNED_SIZE - 1))
#define ERTS_SUPERALIGNED_MASK \
    (~ERTS_INV_SUPERALIGNED_MASK)
#define ERTS_SUPERALIGNED_FLOOR(X) \
    (((UWord) (X)) & ERTS_SUPERALIGNED_MASK)
#define ERTS_SUPERALIGNED_CEILING(X) \
    ERTS_SUPERALIGNED_FLOOR((X) + ERTS_INV_SUPERALIGNED_MASK)
#define ERTS_IS_SUPERALIGNED(X) \
    (((UWord) (X) & ERTS_INV_SUPERALIGNED_MASK) == 0)

#define ERTS_INV_PAGEALIGNED_MASK \
    (erts_page_inv_mask)
#define ERTS_PAGEALIGNED_MASK \
    (~ERTS_INV_PAGEALIGNED_MASK)
#define ERTS_PAGEALIGNED_FLOOR(X) \
    (((UWord) (X)) & ERTS_PAGEALIGNED_MASK)
#define ERTS_PAGEALIGNED_CEILING(X) \
    ERTS_PAGEALIGNED_FLOOR((X) + ERTS_INV_PAGEALIGNED_MASK)
#define ERTS_IS_PAGEALIGNED(X) \
    (((UWord) (X) & ERTS_INV_PAGEALIGNED_MASK) == 0)
#define ERTS_PAGEALIGNED_SIZE \
    (ERTS_INV_PAGEALIGNED_MASK + 1)

#ifndef HAVE_MMAP
#  define HAVE_MMAP 0
#endif
# 103 "sys/common/erl_mmap.h"
#ifndef HAVE_MREMAP
#  define HAVE_MREMAP 0
#endif
# 106 "sys/common/erl_mmap.h"
#if HAVE_MMAP
#  define ERTS_HAVE_OS_MMAP 1
#  define ERTS_HAVE_GENUINE_OS_MMAP 1
#  if HAVE_MREMAP
#    define ERTS_HAVE_OS_MREMAP 1
#  endif
# 112 "sys/common/erl_mmap.h"
#  if defined(MAP_FIXED) && defined(MAP_NORESERVE)
#    define ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION 1
#  endif
# 115 "sys/common/erl_mmap.h"
#endif
# 116 "sys/common/erl_mmap.h"

#ifndef HAVE_VIRTUALALLOC
#  define HAVE_VIRTUALALLOC 0
#endif
# 120 "sys/common/erl_mmap.h"
#if HAVE_VIRTUALALLOC
#  define ERTS_HAVE_OS_MMAP 1
#endif
# 123 "sys/common/erl_mmap.h"

/*#define HARD_DEBUG_MSEG*/
#ifdef HARD_DEBUG_MSEG
#  define HARD_DBG_INSERT_MSEG hard_dbg_insert_mseg
#  define HARD_DBG_REMOVE_MSEG hard_dbg_remove_mseg
void hard_dbg_insert_mseg(void* seg, UWord sz);
void hard_dbg_remove_mseg(void* seg, UWord sz);
#else
# 131 "sys/common/erl_mmap.h"
#  define HARD_DBG_INSERT_MSEG(SEG,SZ)
#  define HARD_DBG_REMOVE_MSEG(SEG,SZ)
#endif
# 134 "sys/common/erl_mmap.h"

#endif /* ERL_MMAP_H__ */
# 136 "sys/common/erl_mmap.h"
# 27 "sys/common/erl_mseg.h" 2

/*
 * We currently only enable mseg_alloc if we got
 * a genuine mmap()/munmap() primitive. It is possible
 * to utilize erts_mmap() withiout a mmap support but
 * alloc_util needs to be prepared before we can do
 * that.
 */
#ifdef ERTS_HAVE_GENUINE_OS_MMAP
#  define HAVE_ERTS_MSEG 1
#  define ERTS_HAVE_MSEG_SUPER_ALIGNED 1
#else
# 39 "sys/common/erl_mseg.h"
#  define HAVE_ERTS_MSEG 0
#  define ERTS_HAVE_MSEG_SUPER_ALIGNED 0
#endif
# 42 "sys/common/erl_mseg.h"

#if ERTS_HAVE_MSEG_SUPER_ALIGNED
#  define MSEG_ALIGN_BITS ERTS_MMAP_SUPERALIGNED_BITS
#else
# 46 "sys/common/erl_mseg.h"
/* If we don't use super aligned multiblock carriers
 * we will mmap with page size alignment (and thus use corresponding
 * align bits).
 *
 * Current implementation needs this to be a constant and
 * only uses this for user dev testing so setting page size
 * to 4096 (12 bits) is fine.
 */
#  define MSEG_ALIGN_BITS       (12)
#endif
# 56 "sys/common/erl_mseg.h"

#if HAVE_ERTS_MSEG

#define MSEG_ALIGNED_SIZE     (1 << MSEG_ALIGN_BITS)

#define ERTS_MSEG_FLG_NONE    ((Uint)(0))
#define ERTS_MSEG_FLG_2POW    ((Uint)(1 << 0))


#define ERTS_MSEG_VSN_STR "0.9"

typedef struct {
    Uint amcbf;
    Uint rmcbf;
    Uint mcs;
    Uint nos;
    ErtsMMapInit mmap;
} ErtsMsegInit_t;

#define ERTS_MSEG_INIT_DEFAULT_INITIALIZER				\
{									\
    4*1024*1024,	/* amcbf: Absolute max cache bad fit	*/	\
    20,			/* rmcbf: Relative max cache bad fit	*/	\
    10,			/* mcs:   Max cache size		*/	\
    1000,		/* cci:   Cache check interval		*/	\
    ERTS_MMAP_INIT_DEFAULT_INITER					\
}

typedef struct {
    int  cache;
    int  preserv;
    UWord abs_shrink_th;
    UWord rel_shrink_th;
    int sched_spec;
#if HALFWORD_HEAP
    int low_mem;
#endif
# 93 "sys/common/erl_mseg.h"
} ErtsMsegOpt_t;

extern const ErtsMsegOpt_t erts_mseg_default_opt;

void *erts_mseg_alloc(ErtsAlcType_t, UWord *, Uint);
void *erts_mseg_alloc_opt(ErtsAlcType_t, UWord *, Uint, const ErtsMsegOpt_t *);
void  erts_mseg_dealloc(ErtsAlcType_t, void *, UWord, Uint);
void  erts_mseg_dealloc_opt(ErtsAlcType_t, void *, UWord, Uint, const ErtsMsegOpt_t *);
void *erts_mseg_realloc(ErtsAlcType_t, void *, UWord, UWord *, Uint);
void *erts_mseg_realloc_opt(ErtsAlcType_t, void *, UWord, UWord *, Uint, const ErtsMsegOpt_t *);
void  erts_mseg_clear_cache(void);
void  erts_mseg_cache_check(void);
Uint  erts_mseg_no( const ErtsMsegOpt_t *);
Uint  erts_mseg_unit_size(void);
void  erts_mseg_init(ErtsMsegInit_t *init);
void  erts_mseg_late_init(void); /* Have to be called after all allocators,
				   threads and timers have been initialized. */
Eterm erts_mseg_info_options(int, int *, void*, Uint **, Uint *);
Eterm erts_mseg_info(int, int *, void*, int, Uint **, Uint *);

#endif /* #if HAVE_ERTS_MSEG */
# 114 "sys/common/erl_mseg.h"

UWord erts_mseg_test(UWord, UWord, UWord, UWord);

#endif /* #ifndef ERL_MSEG_H_ */
# 118 "sys/common/erl_mseg.h"
# 61 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 1 "beam/erl_async.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_ASYNC_H__
#define ERL_ASYNC_H__

#define ERTS_MAX_NO_OF_ASYNC_THREADS 1024
extern int erts_async_max_threads;
#define ERTS_ASYNC_THREAD_MIN_STACK_SIZE 16	/* Kilo words */
#define ERTS_ASYNC_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
extern int erts_async_thread_suggested_stack_size;

#ifdef USE_THREADS

#ifdef ERTS_SMP
/*
 * With smp support we can choose to have, or not to
 * have an async ready queue.
 */
#define ERTS_USE_ASYNC_READY_Q 1
#endif
# 39 "beam/erl_async.h"

#ifndef ERTS_SMP
/* In non-smp case we *need* the async ready queue */
#  undef ERTS_USE_ASYNC_READY_Q
#  define ERTS_USE_ASYNC_READY_Q 1
#endif
# 45 "beam/erl_async.h"

#ifndef ERTS_USE_ASYNC_READY_Q
#  define ERTS_USE_ASYNC_READY_Q 0
#endif
# 49 "beam/erl_async.h"

#if ERTS_USE_ASYNC_READY_Q
int erts_check_async_ready(void *);
int erts_async_ready_clean(void *, void *);
void *erts_get_async_ready_queue(Uint sched_id);
#define ERTS_ASYNC_READY_CLEAN 0
#define ERTS_ASYNC_READY_DIRTY 1
#ifdef ERTS_SMP
#define ERTS_ASYNC_READY_NEED_THR_PRGR 2
#endif
# 59 "beam/erl_async.h"
#endif /* ERTS_USE_ASYNC_READY_Q */
# 60 "beam/erl_async.h"

#endif /* USE_THREADS */
# 62 "beam/erl_async.h"

void erts_init_async(void);
void erts_exit_flush_async(void);


#endif /* ERL_ASYNC_H__ */
# 68 "beam/erl_async.h"
# 62 "beam/erl_process.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 1 "beam/erl_gc.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2007-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERL_GC_H__
#define __ERL_GC_H__

#if defined(ERL_WANT_GC_INTERNALS__) || defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)

/* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */

#if 0 /* expanded by -frewrite-includes */
#include "erl_map.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_gc.h"
# 29 "beam/erl_gc.h"

#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
#  define HARDDEBUG 1
#endif
# 33 "beam/erl_gc.h"

#define IS_MOVED_BOXED(x)	(!is_header((x)))
#define IS_MOVED_CONS(x)	(is_non_value((x)))

#define MOVE_CONS(PTR,CAR,HTOP,ORIG)					\
do {									\
    Eterm gval;								\
									\
    HTOP[0] = CAR;		/* copy car */				\
    HTOP[1] = PTR[1];		/* copy cdr */				\
    gval = make_list(HTOP);	/* new location */			\
    *ORIG = gval;		/* redirect original reference */	\
    PTR[0] = THE_NON_VALUE;	/* store forwarding indicator */	\
    PTR[1] = gval;		/* store forwarding address */		\
    HTOP += 2;			/* update tospace htop */		\
} while(0)

#define MOVE_BOXED(PTR,HDR,HTOP,ORIG)                                   \
do {                                                                    \
    Eterm gval;                                                         \
    Sint nelts;                                                         \
                                                                        \
    ASSERT(is_header(HDR));                                             \
    nelts = header_arity(HDR);                                          \
    switch ((HDR) & _HEADER_SUBTAG_MASK) {                              \
    case SUB_BINARY_SUBTAG: nelts++; break;                             \
    case MAP_SUBTAG:                                                    \
        if (is_flatmap_header(HDR)) nelts+=flatmap_get_size(PTR) + 1;   \
        else nelts += hashmap_bitcount(MAP_HEADER_VAL(HDR));            \
    break;                                                              \
    case FUN_SUBTAG: nelts+=((ErlFunThing*)(PTR))->num_free+1; break;   \
    }                                                                   \
    gval    = make_boxed(HTOP);                                         \
    *ORIG   = gval;                                                     \
    *HTOP++ = HDR;                                                      \
    *PTR++  = gval;                                                     \
    while (nelts--) *HTOP++ = *PTR++;                                   \
} while(0)

#define in_area(ptr,start,nbytes) \
 ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))

#if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG)
int within(Eterm *ptr, Process *p);
#endif
# 78 "beam/erl_gc.h"

ERTS_GLB_INLINE Eterm follow_moved(Eterm term);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm follow_moved(Eterm term)
{
    Eterm* ptr;
    switch (primary_tag(term)) {
    case TAG_PRIMARY_IMMED1:
	break;
    case TAG_PRIMARY_BOXED:
	ptr = boxed_val(term);
	if (IS_MOVED_BOXED(*ptr)) term = *ptr;
	break;
    case TAG_PRIMARY_LIST:
	ptr = list_val(term);
	if (IS_MOVED_CONS(ptr[0])) term = ptr[1];
	break;
    default:
	ASSERT(!"strange tag in follow_moved");
    }
    return term;
}
#endif
# 102 "beam/erl_gc.h"

#endif /* ERL_GC_C__ || HIPE_GC_C__ */
# 104 "beam/erl_gc.h"

/*
 * Global exported
 */

extern Uint erts_test_long_gc_sleep;

typedef struct {
  Uint64 reclaimed;
  Uint64 garbage_cols;
} ErtsGCInfo;

void erts_gc_info(ErtsGCInfo *gcip);
void erts_init_gc(void);
int erts_garbage_collect(struct process*, int, Eterm*, int);
void erts_garbage_collect_hibernate(struct process* p);
Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity);
void erts_garbage_collect_literals(struct process* p, Eterm* literals,
				   Uint lit_size,
				   struct erl_off_heap_header* oh);
Uint erts_next_heap_size(Uint, Uint);
Eterm erts_heap_sizes(struct process* p);

void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*);
void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*);
void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
void erts_free_heap_frags(struct process* p);

#endif /* __ERL_GC_H__ */
# 133 "beam/erl_gc.h"
# 63 "beam/erl_process.h" 2
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 1 "beam/erl_trace.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_TRACE_H__FLAGS__
#define ERL_TRACE_H__FLAGS__
/*
 * NOTE! The bits used for these flags matter. The flag with
 * the least significant bit will take precedence!
 *
 * The "now timestamp" has highest precedence due to
 * compatibility reasons.
 */
#define ERTS_TRACE_FLG_NOW_TIMESTAMP			(1 << 0)
#define ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP	(1 << 1)
#define ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP		(1 << 2)

/*
 * The bits used effects trace flags (of processes and ports)
 * as well as sequential trace flags. If changed make sure
 * these arn't messed up...
 */
#define ERTS_TRACE_TS_TYPE_BITS 			3
#define ERTS_TRACE_TS_TYPE_MASK					\
    ((1 << ERTS_TRACE_TS_TYPE_BITS) - 1)

#define ERTS_TFLGS2TSTYPE(TFLGS)				\
    ((int) (((TFLGS) >> ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) 	\
	    & ERTS_TRACE_TS_TYPE_MASK))
#define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS)				\
    ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)	\
	    & ERTS_TRACE_TS_TYPE_MASK))

#endif /* ERL_TRACE_H__FLAGS__ */
# 51 "beam/erl_trace.h"

#if !defined(ERL_TRACE_H__) && !defined(ERTS_ONLY_INCLUDE_TRACE_FLAGS)
#define ERL_TRACE_H__

struct binary;

/* erl_bif_trace.c */
Eterm erl_seq_trace_info(Process *p, Eterm arg1);
void erts_system_monitor_clear(Process *c_p);
void erts_system_profile_clear(Process *c_p);

/* erl_trace.c */
void erts_init_trace(void);
void erts_trace_check_exiting(Eterm exiting);
Eterm erts_set_system_seq_tracer(Process *c_p,
				 ErtsProcLocks c_p_locks,
				 Eterm new);
Eterm erts_get_system_seq_tracer(void);
void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp);
void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp);
void erts_set_system_monitor(Eterm monitor);
Eterm erts_get_system_monitor(void);
int erts_is_tracer_proc_valid(Process* p);

#ifdef ERTS_SMP
void erts_check_my_tracer_proc(Process *);
void erts_block_sys_msg_dispatcher(void);
void erts_release_sys_msg_dispatcher(void);
void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
					    Eterm,
					    Eterm,
					    ErlHeapFragment *));
void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
#endif
# 85 "beam/erl_trace.h"

void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
void trace_send(Process*, Eterm, Eterm);
void trace_receive(Process*, Eterm);
Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args,
		       int local, Eterm *tracer_pid);
void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
			  Eterm *tracer);
void erts_trace_return_to(Process *p, BeamInstr *pc);
void trace_sched(Process*, Eterm);
void trace_proc(Process*, Process*, Eterm, Eterm);
void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
void save_calls(Process *p, Export *);
void trace_gc(Process *p, Eterm what);
/* port tracing */
void trace_virtual_sched(Process*, Eterm);
void trace_sched_ports(Port *pp, Eterm);
void trace_sched_ports_where(Port *pp, Eterm, Eterm);
void trace_port(Port *, Eterm what, Eterm data);
void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name);

/* system_profile */
void erts_set_system_profile(Eterm profile);
Eterm erts_get_system_profile(void);
void profile_scheduler(Eterm scheduler_id, Eterm);
void profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us);
void profile_runnable_proc(Process* p, Eterm status);
void profile_runnable_port(Port* p, Eterm status);
void erts_system_profile_setup_active_schedulers(void);

/* system_monitor */
void monitor_long_gc(Process *p, Uint time);
void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time);
void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time);
void monitor_large_heap(Process *p);
void monitor_generic(Process *p, Eterm type, Eterm spec);
Uint erts_trace_flag2bit(Eterm flag);
int erts_trace_flags(Eterm List, 
		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);

#ifdef ERTS_SMP
void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)				\
do {									\
    if ((ESDP)->pending_trace_msgs)					\
	erts_send_pending_trace_msgs((ESDP));				\
} while (0)
#else
# 135 "beam/erl_trace.h"
#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)
#endif
# 137 "beam/erl_trace.h"

#define seq_trace_output(token, msg, type, receiver, process) \
seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
#define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \
seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom))
void seq_trace_output_generic(Eterm token, Eterm msg, Uint type, 
			      Eterm receiver, Process *process, Eterm exitfrom);

int seq_trace_update_send(Process *process);

Eterm erts_seq_trace(Process *process, 
		     Eterm atom_type, Eterm atom_true_or_false, 
		     int build_result);

struct trace_pattern_flags {
    unsigned int breakpoint : 1; /* Set if any other is set */
    unsigned int local      : 1; /* Local call trace breakpoint */
    unsigned int meta       : 1; /* Metadata trace breakpoint */
    unsigned int call_count : 1; /* Fast call count breakpoint */
    unsigned int call_time  : 1; /* Fast call time breakpoint */
};
extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
extern int erts_call_time_breakpoint_tracing;
int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
			   struct binary* match_prog_set,
			   struct binary *meta_match_prog_set,
			   int on, struct trace_pattern_flags,
			   Eterm meta_tracer_pid, int is_blocking);
void
erts_get_default_trace_pattern(int *trace_pattern_is_on,
			       struct binary **match_spec,
			       struct binary **meta_match_spec,
			       struct trace_pattern_flags *trace_pattern_flags,
			       Eterm *meta_tracer_pid);
int erts_is_default_trace_enabled(void);
void erts_bif_trace_init(void);
int erts_finish_breakpointing(void);

#endif /* ERL_TRACE_H__ */
# 176 "beam/erl_trace.h"
# 65 "beam/erl_process.h" 2
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 1 "hipe/hipe_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * HiPE-specific process fields
 */
#ifndef HIPE_PROCESS_H
#define HIPE_PROCESS_H

#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc.h"
#endif /* expanded by -frewrite-includes */
# 26 "hipe/hipe_process.h"
# 27 "hipe/hipe_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 27 "hipe/hipe_process.h"
# 1 "beam/export.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __EXPORT_H__
#define __EXPORT_H__

#ifndef __SYS_H__
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/export.h"
# 26 "beam/export.h"
#endif
# 27 "beam/export.h"

#ifndef __INDEX_H__
#if 0 /* expanded by -frewrite-includes */
#include "index.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/export.h"
# 30 "beam/export.h"
#endif
# 31 "beam/export.h"

#if 0 /* expanded by -frewrite-includes */
#include "code_ix.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/export.h"
# 33 "beam/export.h"

/*
** Export entry
*/

typedef struct export
{
    void* addressv[ERTS_NUM_CODE_IX];  /* Pointer to code for function. */

    BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
    /*
     * code[0]: Tagged atom for module.
     * code[1]: Tagged atom for function.
     * code[2]: Arity (untagged integer).
     * code[3]: This entry is 0 unless the 'address' field points to it.
     *          Threaded code instruction to load function
     *		(em_call_error_handler), execute BIF (em_apply_bif),
     *		or a breakpoint instruction (op_i_generic_breakpoint).
     * code[4]: Function pointer to BIF function (for BIFs only),
     *		or pointer to threaded code if the module has an
     *		on_load function that has not been run yet, or pointer
     *          to code for function code[3] is a breakpont instruction.
     *		Otherwise: 0.
     */
    BeamInstr code[5];
} Export;


void init_export_table(void);
void export_info(int, void *);

ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a);
Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity);

Export* erts_export_get_or_make_stub(Eterm, Eterm, unsigned);

Export *export_list(int,ErtsCodeIndex);
int export_list_size(ErtsCodeIndex);
int export_table_sz(void);
int export_entries_sz(void);
Export *export_get(Export*);
void export_start_staging(void);
void export_end_staging(int commit);

extern erts_smp_mtx_t export_staging_lock;
#define export_staging_lock()	erts_smp_mtx_lock(&export_staging_lock)
#define export_staging_unlock()	erts_smp_mtx_unlock(&export_staging_lock)

#if 0 /* expanded by -frewrite-includes */
#include "beam_load.h" /* For em_* extern declarations */ 
#endif /* expanded by -frewrite-includes */
# 81 "beam/export.h"
# 82 "beam/export.h"
#define ExportIsBuiltIn(EntryPtr) 			\
(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \
 ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Export*
erts_active_export_entry(Eterm m, Eterm f, unsigned int a)
{
    extern Export* erts_find_export_entry(Eterm m, Eterm f, unsigned a, ErtsCodeIndex);
    return erts_find_export_entry(m, f, a, erts_active_code_ix());
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 96 "beam/export.h"

#endif /* __EXPORT_H__ */
# 98 "beam/export.h"

# 28 "hipe/hipe_process.h" 2

struct hipe_process_state {
    Eterm *nsp;			/* Native stack pointer. */
    Eterm *nstack;		/* Native stack block start. */
    Eterm *nstend;		/* Native stack block end (start+size). */
    union {
	void (*ncallee)(void);	/* Native code callee (label) to invoke. */
	Eterm closure;		/* Used to pass a closure from native code. */
	Export* callee_exp;     /* Used to pass export entry from native code */
    }u;
    Eterm *nstgraylim;		/* Gray/white stack boundary. */
    Eterm *nstblacklim;		/* Black/gray stack boundary. Must exist if
				   graylim exists. Ignored if no graylim. */
    void (*ngra)(void);		/* Saved original RA from graylim frame. */
#if defined(__i386__) || defined(__x86_64__)
    Eterm *ncsp;		/* Saved C stack pointer. */
#endif
# 45 "hipe/hipe_process.h"
#if defined(__sparc__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
    void (*nra)(void);		/* Native code return address. */
#endif
# 48 "hipe/hipe_process.h"
    unsigned int narity;	/* Arity of BIF call, for stack walks. */
#ifdef NO_FPE_SIGNALS
    double float_result;        /* to be checked for inf/NaN by hipe_emulate_fpe */ 
#endif
# 52 "hipe/hipe_process.h"
#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP)
    void (*bif_callee)(void);   /* When calling BIF's via debug wrapper */
#endif
# 55 "hipe/hipe_process.h"
};

extern void hipe_arch_print_pcb(struct hipe_process_state *p);

static __inline__ void hipe_init_process(struct hipe_process_state *p)
{
    p->nsp = NULL;
    p->nstack = NULL;
    p->nstend = NULL;
    p->nstgraylim = NULL;
    p->nstblacklim = NULL;
    p->ngra = NULL;
#if defined(__sparc__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
    p->nra = NULL;
#endif
# 70 "hipe/hipe_process.h"
    p->narity = 0;
}

static __inline__ void hipe_delete_process(struct hipe_process_state *p)
{
    if (p->nstack)
	erts_free(ERTS_ALC_T_HIPE, (void*)p->nstack);
}

#ifdef ERTS_SMP
struct hipe_process_state_smp {
    int have_receive_locks;
};

static __inline__ void hipe_init_process_smp(struct hipe_process_state_smp *p)
{
    p->have_receive_locks = 0;
}
#endif
# 89 "hipe/hipe_process.h"

#endif /* HIPE_PROCESS_H */
# 91 "hipe/hipe_process.h"
# 69 "beam/erl_process.h" 2
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 1 "beam/erl_thr_progress.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread progress information. Used by lock free algorithms
 *              to determine when all involved threads are guaranteed to
 *              have passed a specific point of execution.
 *
 *              Usage instructions can be found in ert_thr_progress.c
 *
 * Author: 	Rickard Green
 */

#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
#define ERL_THR_PROGRESS_H__TSD_TYPE__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/erl_thr_progress.h"
# 35 "beam/erl_thr_progress.h"

#ifndef ERTS_SMP

#define erts_smp_thr_progress_block() ((void) 0)
#define erts_smp_thr_progress_unblock() ((void) 0)
#define erts_smp_thr_progress_is_blocking() 1

#else /* ERTS_SMP */
# 43 "beam/erl_thr_progress.h"

#define erts_smp_thr_progress_block erts_thr_progress_block
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking

void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);

typedef Uint64 ErtsThrPrgrVal;

#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */

typedef struct {
    ErtsThrPrgrVal next;
    ErtsThrPrgrVal current;
    int chk_next_ix;
    struct {
	int current;
	int waiting;
    } umrefc_ix;
} ErtsThrPrgrLeaderState;

typedef struct {
    int id;
    int is_managed;
    int is_blocking;
#ifdef ERTS_ENABLE_LOCK_CHECK
    int is_delaying; /* managed is always delaying */
#endif
# 73 "beam/erl_thr_progress.h"
    int is_temporary;

    /* --- Part below only for registered threads --- */

    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];

    /* --- Part below only for managed threads --- */

    int leader; /* Needs to be first in the managed threads part */
    int active;
    ErtsThrPrgrVal confirmed;
    ErtsThrPrgrLeaderState leader_state;
} ErtsThrPrgrData;

int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
void erts_thr_progress_fatal_error_wait(SWord timeout);

#endif /* ERTS_SMP */
# 91 "beam/erl_thr_progress.h"

typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
struct ErtsThrPrgrLaterOp_ {
#ifdef ERTS_SMP
    ErtsThrPrgrVal later;
#endif
# 97 "beam/erl_thr_progress.h"
    void (*func)(void *);
    void *data;
    ErtsThrPrgrLaterOp *next;
};

#endif
# 103 "beam/erl_thr_progress.h"

#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
#define ERL_THR_PROGRESS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 107 "beam/erl_thr_progress.h"
# 108 "beam/erl_thr_progress.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 108 "beam/erl_thr_progress.h"
# 109 "beam/erl_thr_progress.h"

#ifdef ERTS_SMP

/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))

extern erts_tsd_key_t erts_thr_prgr_data_key__;

#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t

typedef struct {
    void *arg;
    void (*wakeup)(void *);
    void (*prepare_wait)(void *);
    void (*wait)(void *);
    void (*finalize_wait)(void *);
} ErtsThrPrgrCallbacks;

typedef struct {
    ERTS_THR_PRGR_ATOMIC current;
} ErtsThrPrgr;

typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)

extern ErtsThrPrgr erts_thr_prgr__;

void erts_thr_progress_pre_init(void);
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
					       ErtsThrPrgrCallbacks *,
					       int);
void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
			      ErtsThrPrgrVal value);
int erts_thr_progress_update(ErtsSchedulerData *esdp);
int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
void erts_thr_progress_unmanaged_continue__(int umrefc_ix);

void erts_thr_progress_dbg_print_state(void);

ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);

ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
#endif
# 168 "beam/erl_thr_progress.h"
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
					    ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
}

ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_managed;
}

ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)
{
    if (erts_thr_progress_is_managed_thread())
	return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
    else
	return erts_thr_progress_unmanaged_delay__();
}

ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
{
    ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
	   || erts_thr_progress_is_managed_thread());
    if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
	erts_thr_progress_unmanaged_continue__(handle);
}

#ifdef ERTS_ENABLE_LOCK_CHECK

ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_delaying;
}

#endif
# 233 "beam/erl_thr_progress.h"

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
{
    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
	return ((ErtsThrPrgrVal) 0);
    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
	return ((ErtsThrPrgrVal) 1);
    else
	return val + ((ErtsThrPrgrVal) 2);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData *esdp)
{
    ErtsThrPrgrData *tpd;
    ErtsThrPrgrVal val;
    if (esdp) {
	tpd = &esdp->thr_progress_data;
    managed_thread:
	val = tpd->confirmed;
	ERTS_THR_MEMORY_BARRIER;
    }
    else {
	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
	if (tpd && tpd->is_managed)
	    goto managed_thread;
	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
    }
    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
    return erts_thr_progress_current_to_later__(val);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
    else
	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}

ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
{
    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
	/* May have wrapped... */
	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
	    /*
	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
	     * these ranges 'current' is considered later than 'val0'.
	     */
	    return 1;
	}
    }
    return val1 > val0;
}

ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
{
    if (this == val)
	return 1;
    return erts_thr_progress_has_passed__(this, val);
}

ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
}

ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    if (val1 == val2)
	return 0;
    if (erts_thr_progress_has_passed__(val1, val2))
	return 1;
    else
	return -1;
}	

ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)
{
    ErtsThrPrgrVal current = erts_thr_progress_current();
    return erts_thr_progress_has_reached_this(current, val);
}

#endif
# 326 "beam/erl_thr_progress.h"

#endif /* ERTS_SMP */
# 328 "beam/erl_thr_progress.h"

#endif
# 330 "beam/erl_thr_progress.h"
# 74 "beam/erl_process.h" 2
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 1 "beam/export.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __EXPORT_H__
#define __EXPORT_H__

#ifndef __SYS_H__
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/export.h"
# 26 "beam/export.h"
#endif
# 27 "beam/export.h"

#ifndef __INDEX_H__
#if 0 /* expanded by -frewrite-includes */
#include "index.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/export.h"
# 30 "beam/export.h"
#endif
# 31 "beam/export.h"

#if 0 /* expanded by -frewrite-includes */
#include "code_ix.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/export.h"
# 33 "beam/export.h"

/*
** Export entry
*/

typedef struct export
{
    void* addressv[ERTS_NUM_CODE_IX];  /* Pointer to code for function. */

    BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */
    /*
     * code[0]: Tagged atom for module.
     * code[1]: Tagged atom for function.
     * code[2]: Arity (untagged integer).
     * code[3]: This entry is 0 unless the 'address' field points to it.
     *          Threaded code instruction to load function
     *		(em_call_error_handler), execute BIF (em_apply_bif),
     *		or a breakpoint instruction (op_i_generic_breakpoint).
     * code[4]: Function pointer to BIF function (for BIFs only),
     *		or pointer to threaded code if the module has an
     *		on_load function that has not been run yet, or pointer
     *          to code for function code[3] is a breakpont instruction.
     *		Otherwise: 0.
     */
    BeamInstr code[5];
} Export;


void init_export_table(void);
void export_info(int, void *);

ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a);
Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity);

Export* erts_export_get_or_make_stub(Eterm, Eterm, unsigned);

Export *export_list(int,ErtsCodeIndex);
int export_list_size(ErtsCodeIndex);
int export_table_sz(void);
int export_entries_sz(void);
Export *export_get(Export*);
void export_start_staging(void);
void export_end_staging(int commit);

extern erts_smp_mtx_t export_staging_lock;
#define export_staging_lock()	erts_smp_mtx_lock(&export_staging_lock)
#define export_staging_unlock()	erts_smp_mtx_unlock(&export_staging_lock)

#if 0 /* expanded by -frewrite-includes */
#include "beam_load.h" /* For em_* extern declarations */ 
#endif /* expanded by -frewrite-includes */
# 81 "beam/export.h"
# 82 "beam/export.h"
#define ExportIsBuiltIn(EntryPtr) 			\
(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \
 ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Export*
erts_active_export_entry(Eterm m, Eterm f, unsigned int a)
{
    extern Export* erts_find_export_entry(Eterm m, Eterm f, unsigned a, ErtsCodeIndex);
    return erts_find_export_entry(m, f, a, erts_active_code_ix());
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 96 "beam/export.h"

#endif /* __EXPORT_H__ */
# 98 "beam/export.h"

# 103 "beam/erl_process.h" 2

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 1 "beam/erl_bits.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1999-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERL_BITS_H__
#define __ERL_BITS_H__

/*
 * This structure represents a binary to be matched.
 */

typedef struct erl_bin_match_buffer {
    Eterm orig;			/* Original binary term. */
    byte* base;			/* Current position in binary. */
    Uint offset;		/* Offset in bits. */
    size_t size;		/* Size of binary in bits. */
} ErlBinMatchBuffer;

struct erl_bits_state {
    /*
     * Used for building binaries.
     */
    byte *byte_buf_;
    int byte_buf_len_;
    /*
     * Used for building binaries using the new instruction set.
     */
    byte* erts_current_bin_;	/* Pointer to beginning of current binary. */
    /*
     * Offset in bits into the current binary (new instruction set) or
     * buffer (old instruction set).
     */
    Uint erts_bin_offset_;
    /*
     * Whether the current binary is writable.
     */
     unsigned erts_writable_bin_;
};

typedef struct erl_bin_match_struct{
  Eterm thing_word;
  ErlBinMatchBuffer mb;		/* Present match buffer */
  Eterm save_offset[1];		/* Saved offsets */
} ErlBinMatchState;

#define ERL_BIN_MATCHSTATE_SIZE(_Max) ((sizeof(ErlBinMatchState) + (_Max)*sizeof(Eterm))/sizeof(Eterm)) 
#define HEADER_BIN_MATCHSTATE(_Max) _make_header(ERL_BIN_MATCHSTATE_SIZE((_Max))-1, _TAG_HEADER_BIN_MATCHSTATE)
#define HEADER_NUM_SLOTS(hdr) (header_arity(hdr)-sizeof(ErlBinMatchState)/sizeof(Eterm)+1)

#define make_matchstate(_Ms) make_boxed((Eterm*)(_Ms))  
#define ms_matchbuffer(_Ms) &(((ErlBinMatchState*) boxed_val(_Ms))->mb)


#if defined(ERTS_SMP)
#define ERL_BITS_REENTRANT
#else
# 73 "beam/erl_bits.h"
/* uncomment to test the reentrant API in the non-SMP runtime system */
/* #define ERL_BITS_REENTRANT */
#endif
# 76 "beam/erl_bits.h"

#ifdef ERL_BITS_REENTRANT

/*
 * Reentrant API with the state passed as a parameter.
 * (Except when the current Process* already is a parameter.)
 */
#ifdef ERTS_SMP
/* the state resides in the current process' scheduler data */
#define ERL_BITS_DECLARE_STATEP			struct erl_bits_state *EBS
#define ERL_BITS_RELOAD_STATEP(P)		do{EBS = &(P)->scheduler_data->erl_bits_state;}while(0)
#define ERL_BITS_DEFINE_STATEP(P)		struct erl_bits_state *EBS = &(P)->scheduler_data->erl_bits_state
#else
# 89 "beam/erl_bits.h"
/* reentrant API but with a hidden single global state, for testing only */
extern struct erl_bits_state ErlBitsState_;
#define ERL_BITS_DECLARE_STATEP			struct erl_bits_state *EBS = &ErlBitsState_
#define ERL_BITS_RELOAD_STATEP(P)		do{}while(0)
#define ERL_BITS_DEFINE_STATEP(P)		ERL_BITS_DECLARE_STATEP
#endif
# 95 "beam/erl_bits.h"
#define ErlBitsState				(*EBS)

#define ERL_BITS_PROTO_0			struct erl_bits_state *EBS
#define ERL_BITS_PROTO_1(PARM1)			struct erl_bits_state *EBS, PARM1
#define ERL_BITS_PROTO_2(PARM1,PARM2)		struct erl_bits_state *EBS, PARM1, PARM2
#define ERL_BITS_PROTO_3(PARM1,PARM2,PARM3)	struct erl_bits_state *EBS, PARM1, PARM2, PARM3
#define ERL_BITS_ARGS_0				EBS
#define ERL_BITS_ARGS_1(ARG1)			EBS, ARG1
#define ERL_BITS_ARGS_2(ARG1,ARG2)		EBS, ARG1, ARG2
#define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3)		EBS, ARG1, ARG2, ARG3

#else	/* ERL_BITS_REENTRANT */
# 107 "beam/erl_bits.h"

/*
 * Non-reentrant API with a single global state.
 */
extern struct erl_bits_state ErlBitsState;
#define ERL_BITS_DECLARE_STATEP			/*empty*/
#define ERL_BITS_RELOAD_STATEP(P)		do{}while(0)
#define ERL_BITS_DEFINE_STATEP(P)		/*empty*/

#define ERL_BITS_PROTO_0			void
#define ERL_BITS_PROTO_1(PARM1)			PARM1
#define ERL_BITS_PROTO_2(PARM1,PARM2)		PARM1, PARM2
#define ERL_BITS_PROTO_3(PARM1,PARM2,PARM3)	PARM1, PARM2, PARM3
#define ERL_BITS_ARGS_0				/*empty*/
#define ERL_BITS_ARGS_1(ARG1)			ARG1
#define ERL_BITS_ARGS_2(ARG1,ARG2)		ARG1, ARG2
#define ERL_BITS_ARGS_3(ARG1,ARG2,ARG3)		ARG1, ARG2, ARG3

#endif	/* ERL_BITS_REENTRANT */
# 126 "beam/erl_bits.h"

#define erts_bin_offset		(ErlBitsState.erts_bin_offset_)
#define erts_current_bin	(ErlBitsState.erts_current_bin_)
#define erts_writable_bin       (ErlBitsState.erts_writable_bin_)

#define copy_binary_to_buffer(DstBuffer, DstBufOffset, SrcBuffer, SrcBufferOffset, NumBits) \
  do {											    \
    if (BIT_OFFSET(DstBufOffset) == 0 && (SrcBufferOffset == 0) &&			    \
        (BIT_OFFSET(NumBits)==0)) {							    \
      sys_memcpy(DstBuffer+BYTE_OFFSET(DstBufOffset),					    \
		 SrcBuffer, NBYTES(NumBits));						    \
    } else {										    \
      erts_copy_bits(SrcBuffer, SrcBufferOffset, 1,					    \
        (byte*)DstBuffer, DstBufOffset, 1, NumBits);					    \
    }											    \
  }  while (0)

void erts_init_bits(void);	/* Initialization once. */
#ifdef ERTS_SMP
void erts_bits_init_state(ERL_BITS_PROTO_0);
void erts_bits_destroy_state(ERL_BITS_PROTO_0);
#endif
# 148 "beam/erl_bits.h"


/*
 * NBYTES(x) returns the number of bytes needed to store x bits.
 */

#define NBYTES(x)  (((Uint64)(x) + (Uint64) 7) >> 3) 
#define BYTE_OFFSET(ofs) ((Uint) (ofs) >> 3)
#define BIT_OFFSET(ofs) ((ofs) & 7)

/*
 * Return number of Eterm words needed for allocation with HAlloc(),
 * given a number of bytes.
 */
#define WSIZE(n) ((n + sizeof(Eterm) - 1) / sizeof(Eterm))

/*
 * Binary matching.
 */

Eterm erts_bs_start_match_2(Process *p, Eterm Bin, Uint Max);
Eterm erts_bs_get_integer_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb);
Eterm erts_bs_get_binary_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb);
Eterm erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer* mb);
Eterm erts_bs_get_binary_all_2(Process *p, ErlBinMatchBuffer* mb);

/*
 * Binary construction, new instruction set.
 */

int erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm Integer, Uint num_bits, unsigned flags));
int erts_bs_put_utf8(ERL_BITS_PROTO_1(Eterm Integer));
int erts_bs_put_utf16(ERL_BITS_PROTO_2(Eterm Integer, Uint flags));
int erts_new_bs_put_binary(ERL_BITS_PROTO_2(Eterm Bin, Uint num_bits));
int erts_new_bs_put_binary_all(ERL_BITS_PROTO_2(Eterm Bin, Uint unit));
int erts_new_bs_put_float(Process *c_p, Eterm Float, Uint num_bits, int flags);
void erts_new_bs_put_string(ERL_BITS_PROTO_2(byte* iptr, Uint num_bytes));

Uint erts_bits_bufs_size(void);
Uint32 erts_bs_get_unaligned_uint32(ErlBinMatchBuffer* mb);
void erts_align_utf8_bytes(ErlBinMatchBuffer* mb, byte* buf);
Eterm erts_bs_get_utf8(ErlBinMatchBuffer* mb);
Eterm erts_bs_get_utf16(ErlBinMatchBuffer* mb, Uint flags);
Eterm erts_bs_append(Process* p, Eterm* reg, Uint live, Eterm build_size_term,
		     Uint extra_words, Uint unit);
Eterm erts_bs_private_append(Process* p, Eterm bin, Eterm sz, Uint unit);
Eterm erts_bs_init_writable(Process* p, Eterm sz);

/*
 * Common utilities.
 */
void erts_copy_bits(byte* src, size_t soffs, int sdir,
		    byte* dst, size_t doffs,int ddir, size_t n);        
int erts_cmp_bits(byte* a_ptr, size_t a_offs, byte* b_ptr, size_t b_offs, size_t size); 

/*
 * Flags for bs_get_* / bs_put_* / bs_init* instructions.
 */

#define BSF_ALIGNED 1		/* Field is guaranteed to be byte-aligned. */
#define BSF_LITTLE 2		/* Field is little-endian (otherwise big-endian). */
#define BSF_SIGNED 4		/* Field is signed (otherwise unsigned). */
#define BSF_EXACT 8		/* Size in bs_init is exact. */
#define BSF_NATIVE 16		/* Native endian. */

#endif /* __ERL_BITS_H__ */
# 214 "beam/erl_bits.h"
# 127 "beam/erl_process.h" 2
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 1 "beam/erl_process_lock.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2007-2012. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */


/*
 * Description:	Impementation of Erlang process locks.
 *
 * Author: 	Rickard Green
 */

#ifndef ERTS_PROC_LOCK_TYPE__
#define ERTS_PROC_LOCK_TYPE__

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_PROC_LOCK_DEBUG
#endif
# 34 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_count.h"
#endif /* expanded by -frewrite-includes */
# 36 "beam/erl_process_lock.h"
# 37 "beam/erl_process_lock.h"
#endif
# 38 "beam/erl_process_lock.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 39 "beam/erl_process_lock.h"
# 40 "beam/erl_process_lock.h"

#if defined(VALGRIND) || defined(ETHR_DISABLE_NATIVE_IMPLS)
#  define ERTS_PROC_LOCK_OWN_IMPL 0
#else
# 44 "beam/erl_process_lock.h"
#  define ERTS_PROC_LOCK_OWN_IMPL 1
#endif
# 46 "beam/erl_process_lock.h"

#define ERTS_PROC_LOCK_ATOMIC_IMPL 0
#define ERTS_PROC_LOCK_SPINLOCK_IMPL 0
#define ERTS_PROC_LOCK_MUTEX_IMPL 0

#if !ERTS_PROC_LOCK_OWN_IMPL
#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 1
#else
# 54 "beam/erl_process_lock.h"
#define ERTS_PROC_LOCK_RAW_MUTEX_IMPL 0

#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
#  undef ERTS_PROC_LOCK_ATOMIC_IMPL
#  define ERTS_PROC_LOCK_ATOMIC_IMPL 1
#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS)
# 60 "beam/erl_process_lock.h"
#  undef ERTS_PROC_LOCK_SPINLOCK_IMPL
#  define ERTS_PROC_LOCK_SPINLOCK_IMPL 1
#else
# 63 "beam/erl_process_lock.h"
#  undef ERTS_PROC_LOCK_MUTEX_IMPL
#  define ERTS_PROC_LOCK_MUTEX_IMPL 1
#endif
# 66 "beam/erl_process_lock.h"

#endif
# 68 "beam/erl_process_lock.h"

#define ERTS_PROC_LOCK_MAX_BIT 4

typedef erts_aint32_t ErtsProcLocks;

typedef struct erts_proc_lock_t_ {
#if ERTS_PROC_LOCK_OWN_IMPL
#if ERTS_PROC_LOCK_ATOMIC_IMPL
    erts_smp_atomic32_t flags;
#else
# 78 "beam/erl_process_lock.h"
    ErtsProcLocks flags;
#endif
# 80 "beam/erl_process_lock.h"
    erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1];
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt_main;
    erts_lcnt_lock_t lcnt_link;
    erts_lcnt_lock_t lcnt_msgq;
    erts_lcnt_lock_t lcnt_btm;
    erts_lcnt_lock_t lcnt_status;
#endif
# 88 "beam/erl_process_lock.h"
#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 89 "beam/erl_process_lock.h"
    erts_mtx_t main;
    erts_mtx_t link;
    erts_mtx_t msgq;
    erts_mtx_t btm;
    erts_mtx_t status;
#else
# 95 "beam/erl_process_lock.h"
#  error "no implementation"
#endif
# 97 "beam/erl_process_lock.h"
#ifdef ERTS_PROC_LOCK_DEBUG
    erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1];
#endif
# 100 "beam/erl_process_lock.h"
} erts_proc_lock_t;

/* Process lock flags */

/*
 * Main lock:
 *   The main lock is held by the scheduler running a process. It
 *   is used to protect all fields in the process structure except
 *   for those fields protected by other process locks (follows).
 */
#define ERTS_PROC_LOCK_MAIN		(((ErtsProcLocks) 1) << 0)

/*
 * Link lock:
 *   Protects the following fields in the process structure:
 *   * nlinks
 *   * monitors
 *   * suspend_monitors
 */
#define ERTS_PROC_LOCK_LINK		(((ErtsProcLocks) 1) << 1)

/*
 * Message queue lock:
 *   Protects the following fields in the process structure:
 *   * msg_inq
 */
#define ERTS_PROC_LOCK_MSGQ		(((ErtsProcLocks) 1) << 2)

/*
 * Bif timer lock:
 *   Protects the following fields in the process structure:
 *   * bif_timers
 */
#define ERTS_PROC_LOCK_BTM		(((ErtsProcLocks) 1) << 3)

/*
 * Status lock:
 *   Protects the following fields in the process structure:
 *   * pending_suspenders
 *   * suspendee
 *   * ...
 */
#define ERTS_PROC_LOCK_STATUS		(((ErtsProcLocks) 1) << ERTS_PROC_LOCK_MAX_BIT)

/*
 * Special fields:
 *
 *   The following fields are read only and can be read if at
 *   least one process lock (whichever one doesn't matter)
 *   is held, or if the process structure is guaranteed not to
 *   disappear by other means (e.g. pix lock is held):
 *     * id
 *
 *   The following fields are only allowed to be written if
 *   all process locks are held, and are allowed to be read if
 *   at least one process lock (whichever one doesn't matter)
 *   is held:
 *     * tracer_proc
 *     * tracer_flags
 *
 *   The following fields are only allowed to be accessed if
 *   both the schedule queue lock and at least one process lock
 *   (whichever one doesn't matter) are held:
 *     * prio
 *     * next
 *     * scheduler_flags
 */

/*
 * Other rules regarding process locking:
 *
 * Exiting processes:
 *   When changing state to exiting (ERTS_PSFLG_EXITING) on a process,
 *   you are required to take all process locks (ERTS_PROC_LOCKS_ALL).
 *   Thus, by holding at least one process lock (whichever one doesn't
 *   matter) you are guaranteed that the process won't exit until the
 *   lock you are holding has been released.
 *
 * Lock order:
 *   Process locks with low numeric values has to be locked before
 *   process locks with high numeric values. E.g., main locks has
 *   to be locked before message queue locks.
 *
 *   When process locks with the same numeric value are to be locked
 *   on multiple processes, locks on processes with low process ids
 *   have to be locked before locks on processes with high process
 *   ids. E.g., if the main and the message queue locks are to be
 *   locked on processes p1 and p2 and p1->common.id < p2->common.id,
 *   then locks should be locked in the following order:
 *     1. main lock on p1
 *     2. main lock on p2
 *     3. message queue lock on p1
 *     4. message queue lock on p2
 */

/* Other lock flags */
#define ERTS_PROC_LOCK_WAITER_SHIFT (ERTS_PROC_LOCK_MAX_BIT + 1)


/* ERTS_PROC_LOCKS_* are combinations of process locks */

#define ERTS_PROC_LOCKS_MSG_RECEIVE	(ERTS_PROC_LOCK_MSGQ		\
					 | ERTS_PROC_LOCK_STATUS)
#define ERTS_PROC_LOCKS_MSG_SEND	(ERTS_PROC_LOCK_MSGQ		\
					 | ERTS_PROC_LOCK_STATUS)
#define ERTS_PROC_LOCKS_XSIG_SEND	ERTS_PROC_LOCK_STATUS

#define ERTS_PROC_LOCKS_ALL \
  ((((ErtsProcLocks) 1) << (ERTS_PROC_LOCK_MAX_BIT + 1)) - 1)

#define ERTS_PROC_LOCKS_ALL_MINOR	(ERTS_PROC_LOCKS_ALL \
					 & ~ERTS_PROC_LOCK_MAIN)


#define ERTS_PIX_LOCKS_BITS		10
#define ERTS_NO_OF_PIX_LOCKS		(1 << ERTS_PIX_LOCKS_BITS)


#endif /* #ifndef ERTS_PROC_LOCK_TYPE__ */
# 219 "beam/erl_process_lock.h"

#ifndef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#ifndef ERTS_PROC_LOCK_LOCK_CHECK__
#define  ERTS_PROC_LOCK_LOCK_CHECK__

/* Lock counter implemetation */

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_proc_lock__(P,I,L) erts_smp_proc_lock_x__(P,I,L,__FILE__,__LINE__)
#define erts_smp_proc_lock(P,L) erts_smp_proc_lock_x(P,L,__FILE__,__LINE__)
#endif
# 230 "beam/erl_process_lock.h"

#if defined(ERTS_SMP) && defined (ERTS_ENABLE_LOCK_COUNT)

void erts_lcnt_proc_lock_init(Process *p);
void erts_lcnt_proc_lock_destroy(Process *p);
void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks);
void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line);
void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks);
void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks);
void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res);

void erts_lcnt_enable_proc_lock_count(int enable);

#endif /* ERTS_ENABLE_LOCK_COUNT*/
# 244 "beam/erl_process_lock.h"



/* --- Process lock checking ----------------------------------------------- */

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
#define ERTS_SMP_CHK_NO_PROC_LOCKS \
  erts_proc_lc_chk_no_proc_locks(__FILE__, __LINE__)
#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P) \
  erts_proc_lc_chk_only_proc_main((P))
void erts_proc_lc_lock(Process *p, ErtsProcLocks locks,
		       char *file, unsigned int line);
void erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
			  char *file, unsigned int line);
void erts_proc_lc_unlock(Process *p, ErtsProcLocks locks);
void erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks);
void erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks);
void erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks);
void erts_proc_lc_chk_only_proc_main(Process *p);
void erts_proc_lc_chk_no_proc_locks(char *file, int line);
ErtsProcLocks erts_proc_lc_my_proc_locks(Process *p);
int erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks);
void erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks,
			       char* file, unsigned int line);
void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks);
#else
# 270 "beam/erl_process_lock.h"
#define ERTS_SMP_CHK_NO_PROC_LOCKS
#define ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(P)
#endif
# 273 "beam/erl_process_lock.h"

#endif /* #ifndef ERTS_PROC_LOCK_LOCK_CHECK__ */
# 275 "beam/erl_process_lock.h"
#endif /* #ifndef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ */
# 276 "beam/erl_process_lock.h"

#if !defined(ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__) \
    && !defined(ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__)
#ifndef ERTS_PROCESS_LOCK_H__
#define ERTS_PROCESS_LOCK_H__

#ifdef ERTS_SMP

typedef struct {
    union {
	erts_mtx_t mtx;
	char buf[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))];
    } u;
} erts_pix_lock_t;

#define ERTS_PID2PIXLOCK(PID) \
    (&erts_pix_locks[(internal_pid_data((PID)) & ((1 << ERTS_PIX_LOCKS_BITS) - 1))])

#if ERTS_PROC_LOCK_OWN_IMPL

#if ERTS_PROC_LOCK_ATOMIC_IMPL

#define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \
  ((ErtsProcLocks) erts_smp_atomic32_read_band_nob(&(L)->flags, \
						   (erts_aint32_t) (MSK)))
#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) \
  ((ErtsProcLocks) erts_smp_atomic32_read_bor_acqb(&(L)->flags, \
						   (erts_aint32_t) (MSK)))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \
						  (erts_aint32_t) (NEW), \
						  (erts_aint32_t) (EXPECTED)))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
  ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \
						  (erts_aint32_t) (NEW), \
						  (erts_aint32_t) (EXPECTED)))
#define ERTS_PROC_LOCK_FLGS_READ_(L) \
  ((ErtsProcLocks) erts_smp_atomic32_read_nob(&(L)->flags))

#else /* no opt atomic ops */
# 316 "beam/erl_process_lock.h"

ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *,
							ErtsProcLocks);
ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *,
						       ErtsProcLocks);
ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *,
							   ErtsProcLocks,
							   ErtsProcLocks);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsProcLocks
erts_proc_lock_flags_band(erts_proc_lock_t *lck, ErtsProcLocks mask)
{
    ErtsProcLocks res = lck->flags;
    lck->flags &= mask;
    return res;
}

ERTS_GLB_INLINE ErtsProcLocks
erts_proc_lock_flags_bor(erts_proc_lock_t *lck, ErtsProcLocks mask)
{
    ErtsProcLocks res = lck->flags;
    lck->flags |= mask;
    return res;
}

ERTS_GLB_INLINE ErtsProcLocks
erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new,
                             ErtsProcLocks expected)
{
    ErtsProcLocks res = lck->flags;
    if (res == expected)
        lck->flags = new;
    return res;
}

#endif
# 354 "beam/erl_process_lock.h"

#define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK))
#define ERTS_PROC_LOCK_FLGS_BOR_ACQB_(L, MSK) erts_proc_lock_flags_bor((L), (MSK))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \
  erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \
  erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED))
#define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags)

#endif /* end no opt atomic ops */
# 364 "beam/erl_process_lock.h"
#endif /* ERTS_PROC_LOCK_OWN_IMPL */
# 365 "beam/erl_process_lock.h"

extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];

void erts_init_proc_lock(int cpus);
void erts_proc_lock_prepare_proc_lock_waiter(void);
#if ERTS_PROC_LOCK_OWN_IMPL
void erts_proc_lock_failed(Process *,
			   erts_pix_lock_t *,
			   ErtsProcLocks,
			   ErtsProcLocks);
void erts_proc_unlock_failed(Process *,
			     erts_pix_lock_t *,
			     ErtsProcLocks);
#endif
# 379 "beam/erl_process_lock.h"

ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *);
ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *);
ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *);

ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p,
							  ErtsProcLocks locks);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *,
					    erts_pix_lock_t *,
					    ErtsProcLocks,
					    char *file, unsigned int line);
#else
# 392 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *,
					  erts_pix_lock_t *,
					  ErtsProcLocks);
#endif
# 396 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_unlock__(Process *,
					    erts_pix_lock_t *,
					    ErtsProcLocks);
ERTS_GLB_INLINE int erts_smp_proc_trylock__(Process *,
					    erts_pix_lock_t *,
					    ErtsProcLocks);

#ifdef ERTS_PROC_LOCK_DEBUG
ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int);
#endif
# 406 "beam/erl_process_lock.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck)
{
    ERTS_LC_ASSERT(pixlck);
    erts_mtx_lock(&pixlck->u.mtx);
}

ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck)
{
    ERTS_LC_ASSERT(pixlck);
    erts_mtx_unlock(&pixlck->u.mtx);
}

ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck)
{
    return erts_lc_mtx_is_locked(&pixlck->u.mtx);
}

/*
 * Helper function for erts_smp_proc_lock__ and erts_smp_proc_trylock__.
 *
 * Attempts to grab all of 'locks' simultaneously.
 *
 * On success, returns zero.
 *
 * On failure, returns the p->locks at the moment it tried to grab them,
 * at least some of which will intersect with 'locks', so it is nonzero.
 *
 * This assumes p's pix lock is held on entry if !ERTS_PROC_LOCK_ATOMIC_IMPL.
 * Does not release the pix lock.
 */
ERTS_GLB_INLINE ErtsProcLocks
erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks)
{
#if ERTS_PROC_LOCK_OWN_IMPL
    ErtsProcLocks expct_lflgs = 0;

    while (1) {
        ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock,
								expct_lflgs | locks,
								expct_lflgs);
        if (ERTS_LIKELY(lflgs == expct_lflgs)) {
            /* We successfully grabbed all locks. */
            return 0;
        }

        if (lflgs & locks) {
            /* Some locks we need are locked, give up. */
            return lflgs;
        }

        /* cmpxchg failed, try again (should be rare). */
        expct_lflgs = lflgs;
    }

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 464 "beam/erl_process_lock.h"

    if (locks & ERTS_PROC_LOCK_MAIN)
	if (erts_mtx_trylock(&p->lock.main) == EBUSY)
	    goto busy_main;
    if (locks & ERTS_PROC_LOCK_LINK)
	if (erts_mtx_trylock(&p->lock.link) == EBUSY)
	    goto busy_link;
    if (locks & ERTS_PROC_LOCK_MSGQ)
	if (erts_mtx_trylock(&p->lock.msgq) == EBUSY)
	    goto busy_msgq;
    if (locks & ERTS_PROC_LOCK_BTM)
	if (erts_mtx_trylock(&p->lock.btm) == EBUSY)
	    goto busy_btm;
    if (locks & ERTS_PROC_LOCK_STATUS)
	if (erts_mtx_trylock(&p->lock.status) == EBUSY)
	    goto busy_status;

    return 0;

busy_status:
    if (locks & ERTS_PROC_LOCK_BTM)
	erts_mtx_unlock(&p->lock.btm);
busy_btm:
    if (locks & ERTS_PROC_LOCK_MSGQ)
	erts_mtx_unlock(&p->lock.msgq);
busy_msgq:
    if (locks & ERTS_PROC_LOCK_LINK)
	erts_mtx_unlock(&p->lock.link);
busy_link:
    if (locks & ERTS_PROC_LOCK_MAIN)
	erts_mtx_unlock(&p->lock.main);
busy_main:

    return EBUSY;
#endif
# 499 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_proc_lock_x__(Process *p,
		     erts_pix_lock_t *pix_lck,
		     ErtsProcLocks locks,
		     char *file, unsigned int line)
#else
# 508 "beam/erl_process_lock.h"
erts_smp_proc_lock__(Process *p,
		     erts_pix_lock_t *pix_lck,
		     ErtsProcLocks locks)
#endif
# 512 "beam/erl_process_lock.h"
{
#if ERTS_PROC_LOCK_OWN_IMPL

    ErtsProcLocks old_lflgs;
#if !ERTS_PROC_LOCK_ATOMIC_IMPL
    erts_pix_lock(pix_lck);
#endif
# 519 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_lock(&(p->lock), locks);
#endif
# 523 "beam/erl_process_lock.h"

    ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_proc_lc_lock(p, locks, file, line);
#endif
# 529 "beam/erl_process_lock.h"

    old_lflgs = erts_smp_proc_raw_trylock__(p, locks);

    if (old_lflgs != 0) {
	/*
         * There is lock contention, so let erts_proc_lock_failed() deal
         * with it. Note that erts_proc_lock_failed() returns with
         * pix_lck unlocked.
         */
	erts_proc_lock_failed(p, pix_lck, locks, old_lflgs);
    }

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
    else {
	ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
	erts_pix_unlock(pix_lck);
    }
#endif
# 547 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line);
#endif
# 551 "beam/erl_process_lock.h"

#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 1);
#endif
# 555 "beam/erl_process_lock.h"

#if ERTS_PROC_LOCK_ATOMIC_IMPL
    ETHR_COMPILER_BARRIER;
#endif
# 559 "beam/erl_process_lock.h"

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 561 "beam/erl_process_lock.h"
    if (locks & ERTS_PROC_LOCK_MAIN)
	erts_mtx_lock(&p->lock.main);
    if (locks & ERTS_PROC_LOCK_LINK)
	erts_mtx_lock(&p->lock.link);
    if (locks & ERTS_PROC_LOCK_MSGQ)
	erts_mtx_lock(&p->lock.msgq);
    if (locks & ERTS_PROC_LOCK_BTM)
	erts_mtx_lock(&p->lock.btm);
    if (locks & ERTS_PROC_LOCK_STATUS)
	erts_mtx_lock(&p->lock.status);

#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 1);
#endif
# 575 "beam/erl_process_lock.h"

#endif
# 577 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void
erts_smp_proc_unlock__(Process *p,
		       erts_pix_lock_t *pix_lck,
		       ErtsProcLocks locks)
{
#if ERTS_PROC_LOCK_OWN_IMPL
    ErtsProcLocks old_lflgs;

#if ERTS_PROC_LOCK_ATOMIC_IMPL
    ETHR_COMPILER_BARRIER;
#endif
# 590 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_unlock(&(p->lock), locks);
#endif
# 594 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_proc_lc_unlock(p, locks);
#endif
# 598 "beam/erl_process_lock.h"
#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 0);
#endif
# 601 "beam/erl_process_lock.h"

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
    erts_pix_lock(pix_lck);
#endif
# 605 "beam/erl_process_lock.h"

    old_lflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock);

    ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);
    ERTS_LC_ASSERT(locks == (old_lflgs & locks));

    while (1) {
        /*
         * We'll atomically unlock every lock that has no waiter.
         * If any locks with waiters remain we'll let
         * erts_proc_unlock_failed() deal with them.
         */
        ErtsProcLocks wait_locks =
            (old_lflgs >> ERTS_PROC_LOCK_WAITER_SHIFT) & locks;

        /* What p->lock will look like with all non-waited locks released. */
        ErtsProcLocks want_lflgs = old_lflgs & (wait_locks | ~locks);

        if (want_lflgs != old_lflgs) {
            ErtsProcLocks new_lflgs =
                ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs);

            if (new_lflgs != old_lflgs) {
                /* cmpxchg failed, try again. */
                old_lflgs = new_lflgs;
                continue;
            }
        }

        /* We have successfully unlocked every lock with no waiter. */

        if (want_lflgs & locks) {
            /* Locks with waiters remain. */
            /* erts_proc_unlock_failed() returns with pix_lck unlocked. */
            erts_proc_unlock_failed(p, pix_lck, want_lflgs & locks);
        }
        else {
#if !ERTS_PROC_LOCK_ATOMIC_IMPL
            erts_pix_unlock(pix_lck);
#endif
# 645 "beam/erl_process_lock.h"
        }

        break;
    }

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 651 "beam/erl_process_lock.h"

#ifdef ERTS_PROC_LOCK_DEBUG
    erts_proc_lock_op_debug(p, locks, 0);
#endif
# 655 "beam/erl_process_lock.h"

    if (locks & ERTS_PROC_LOCK_STATUS)
	erts_mtx_unlock(&p->lock.status);
    if (locks & ERTS_PROC_LOCK_BTM)
	erts_mtx_unlock(&p->lock.btm);
    if (locks & ERTS_PROC_LOCK_MSGQ)
	erts_mtx_unlock(&p->lock.msgq);
    if (locks & ERTS_PROC_LOCK_LINK)
	erts_mtx_unlock(&p->lock.link);
    if (locks & ERTS_PROC_LOCK_MAIN)
	erts_mtx_unlock(&p->lock.main);
#endif
# 667 "beam/erl_process_lock.h"

}

ERTS_GLB_INLINE int
erts_smp_proc_trylock__(Process *p,
			erts_pix_lock_t *pix_lck,
			ErtsProcLocks locks)
{
#if ERTS_PROC_LOCK_OWN_IMPL
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0);
    if (erts_proc_lc_trylock_force_busy(p, locks)) {
	res = EBUSY; /* Make sure caller can handle the situation without
			causing a lock order violation to occur */
    }
    else
#endif
# 686 "beam/erl_process_lock.h"
    {

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
	erts_pix_lock(pix_lck);
#endif
# 691 "beam/erl_process_lock.h"

	if (erts_smp_proc_raw_trylock__(p, locks) != 0) {
	    /* Didn't get all locks... */
	    res = EBUSY;

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
	    erts_pix_unlock(pix_lck);
#endif
# 699 "beam/erl_process_lock.h"
	}
	else {
	    res = 0;

	    ERTS_LC_ASSERT(locks
			   == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));

#if !ERTS_PROC_LOCK_ATOMIC_IMPL
	    erts_pix_unlock(pix_lck);
#endif
# 709 "beam/erl_process_lock.h"

#ifdef ERTS_PROC_LOCK_DEBUG
	    erts_proc_lock_op_debug(p, locks, 1);
#endif
# 713 "beam/erl_process_lock.h"
	}
    }
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_proc_trylock(&(p->lock), locks, res);
#endif
# 718 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_proc_lc_trylock(p, locks, res == 0, __FILE__, __LINE__);
#endif
# 722 "beam/erl_process_lock.h"

#if ERTS_PROC_LOCK_ATOMIC_IMPL
    ETHR_COMPILER_BARRIER;
#endif
# 726 "beam/erl_process_lock.h"
    return res;

#elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
# 729 "beam/erl_process_lock.h"
    if (erts_smp_proc_raw_trylock__(p, locks) != 0)
	return EBUSY;
    else {
#ifdef ERTS_PROC_LOCK_DEBUG
	erts_proc_lock_op_debug(p, locks, 1);
#endif
# 735 "beam/erl_process_lock.h"
	return 0;
    }
#endif
# 738 "beam/erl_process_lock.h"
}

#ifdef ERTS_PROC_LOCK_DEBUG
ERTS_GLB_INLINE void 
erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked)
{
    int i;
    for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) {
	ErtsProcLocks lock = ((ErtsProcLocks) 1) << i;
	if (locks & lock) {
	    erts_aint32_t lock_count;
	    if (locked) {
		lock_count = erts_smp_atomic32_inc_read_nob(&p->lock.locked[i]);
		ERTS_LC_ASSERT(lock_count == 1);
	    }
	    else {
		lock_count = erts_smp_atomic32_dec_read_nob(&p->lock.locked[i]);
		ERTS_LC_ASSERT(lock_count == 0);
	    }
	}
    }
}
#endif
# 761 "beam/erl_process_lock.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 763 "beam/erl_process_lock.h"

#endif /* ERTS_SMP */
# 765 "beam/erl_process_lock.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_proc_lock_x(Process *, ErtsProcLocks, char *file, unsigned int line);
#else
# 769 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_lock(Process *, ErtsProcLocks);
#endif
# 771 "beam/erl_process_lock.h"
ERTS_GLB_INLINE void erts_smp_proc_unlock(Process *, ErtsProcLocks);
ERTS_GLB_INLINE int erts_smp_proc_trylock(Process *, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_inc_refc(Process *);
ERTS_GLB_INLINE void erts_proc_dec_refc(Process *);
ERTS_GLB_INLINE void erts_proc_add_refc(Process *, Sint);
ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_proc_lock_x(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
#else
# 785 "beam/erl_process_lock.h"
erts_smp_proc_lock(Process *p, ErtsProcLocks locks)
#endif 
# 787 "beam/erl_process_lock.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_smp_proc_lock_x__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
			 NULL,
#else
# 793 "beam/erl_process_lock.h"
			 ERTS_PID2PIXLOCK(p->common.id),
#endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
# 795 "beam/erl_process_lock.h"
			 locks, file, line);
#elif defined(ERTS_SMP)
# 797 "beam/erl_process_lock.h"
    erts_smp_proc_lock__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
			 NULL,
#else
# 801 "beam/erl_process_lock.h"
			 ERTS_PID2PIXLOCK(p->common.id),
#endif /*ERTS_PROC_LOCK_ATOMIC_IMPL*/
# 803 "beam/erl_process_lock.h"
			 locks);
#endif /*ERTS_SMP*/
# 805 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void
erts_smp_proc_unlock(Process *p, ErtsProcLocks locks)
{
#ifdef ERTS_SMP
    erts_smp_proc_unlock__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
			   NULL,
#else
# 815 "beam/erl_process_lock.h"
			   ERTS_PID2PIXLOCK(p->common.id),
#endif
# 817 "beam/erl_process_lock.h"
			   locks);
#endif
# 819 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE int
erts_smp_proc_trylock(Process *p, ErtsProcLocks locks)
{
#ifndef ERTS_SMP
    return 0;
#else
# 827 "beam/erl_process_lock.h"
    return erts_smp_proc_trylock__(p,
#if ERTS_PROC_LOCK_ATOMIC_IMPL
				   NULL,
#else
# 831 "beam/erl_process_lock.h"
				   ERTS_PID2PIXLOCK(p->common.id),
#endif
# 833 "beam/erl_process_lock.h"
				   locks);
#endif
# 835 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void erts_proc_inc_refc(Process *p)
{
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    erts_ptab_atmc_inc_refc(&p->common);
#else
# 843 "beam/erl_process_lock.h"
    erts_ptab_inc_refc(&p->common);
#endif
# 845 "beam/erl_process_lock.h"
}

ERTS_GLB_INLINE void erts_proc_dec_refc(Process *p)
{
    Sint referred;
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    referred = erts_ptab_atmc_dec_test_refc(&p->common);
#else
# 854 "beam/erl_process_lock.h"
    referred = erts_ptab_dec_test_refc(&p->common);
#endif
# 856 "beam/erl_process_lock.h"
    if (!referred) {
	ASSERT(ERTS_PROC_IS_EXITING(p));
	erts_free_proc(p);
    }
}

ERTS_GLB_INLINE void erts_proc_add_refc(Process *p, Sint add_refc)
{
    Sint referred;
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    referred = erts_ptab_atmc_add_test_refc(&p->common, add_refc);
#else
# 869 "beam/erl_process_lock.h"
    referred = erts_ptab_add_test_refc(&p->common, add_refc);
#endif
# 871 "beam/erl_process_lock.h"
    if (!referred) {
	ASSERT(ERTS_PROC_IS_EXITING(p));
	erts_free_proc(p);
    }
}

ERTS_GLB_INLINE Sint erts_proc_read_refc(Process *p)
{
    ASSERT(!(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_PROXY));
#ifdef ERTS_SMP
    return erts_ptab_atmc_read_refc(&p->common);
#else
# 883 "beam/erl_process_lock.h"
    return erts_ptab_read_refc(&p->common);
#endif
# 885 "beam/erl_process_lock.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 888 "beam/erl_process_lock.h"

#ifdef ERTS_SMP
void erts_proc_lock_init(Process *);
void erts_proc_lock_fin(Process *);
void erts_proc_safelock(Process *a_proc,
			ErtsProcLocks a_have_locks,
			ErtsProcLocks a_need_locks,
			Process *b_proc,
			ErtsProcLocks b_have_locks,
			ErtsProcLocks b_need_locks);
#endif
# 899 "beam/erl_process_lock.h"

/*
 * --- Process table lookup ------------------------------------------------
 *
 * erts_pid2proc() and friends looks up the process structure of a pid
 * and at the same time acquires process locks in the smp case. Locks
 * on currently executing process and looked up process are taken according
 * to the lock order, i.e., locks on currently executing process may have
 * been released and reacquired.
 *
 * erts_pid2proc_opt() currently accepts the following flags:
 *   ERTS_P2P_FLG_ALLOW_OTHER_X    Lookup process even if it currently
 *                                 is exiting.
 */

#define ERTS_P2P_FLG_ALLOW_OTHER_X	(1 <<  0)
#define ERTS_P2P_FLG_TRY_LOCK		(1 <<  1)
#define ERTS_P2P_FLG_INC_REFC		(1 <<  2)

#define ERTS_PROC_LOCK_BUSY ((Process *) &erts_invalid_process)

#define erts_pid2proc(PROC, HL, PID, NL) \
  erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0)


ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid);

#ifndef ERTS_SMP
ERTS_GLB_INLINE
#endif
# 931 "beam/erl_process_lock.h"
Process *erts_pid2proc_opt(Process *, ErtsProcLocks, Eterm, ErtsProcLocks, int);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Process *erts_pix2proc(int ix)
{
    Process *proc;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_proc));
    proc = (Process *) erts_ptab_pix2intptr_nob(&erts_proc, ix);
    return proc == ERTS_PROC_LOCK_BUSY ? NULL : proc;
}

ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid)
{
    Process *proc;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_pid(pid))
	return NULL;

    proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc,
						 internal_pid_index(pid));
    if (proc && proc->common.id != pid)
	return NULL;
    return proc;
}

ERTS_GLB_INLINE Process *erts_proc_lookup(Eterm pid)
{
    Process *proc = erts_proc_lookup_raw(pid);
    if (proc && ERTS_PROC_IS_EXITING(proc))
	return NULL;
    return proc;
}

#ifndef ERTS_SMP
ERTS_GLB_INLINE Process *
erts_pid2proc_opt(Process *c_p_unused,
		  ErtsProcLocks c_p_have_locks_unused,
		  Eterm pid,
		  ErtsProcLocks pid_need_locks_unused,
		  int flags)
{
    Process *proc = erts_proc_lookup_raw(pid);
    if (!proc)
	return NULL;
    if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
	&& ERTS_PROC_IS_EXITING(proc))
	return NULL;
    if (flags & ERTS_P2P_FLG_INC_REFC)
	erts_proc_inc_refc(proc);
    return proc;
}
#endif /* !ERTS_SMP */
# 986 "beam/erl_process_lock.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 988 "beam/erl_process_lock.h"

#endif /* #ifndef ERTS_PROCESS_LOCK_H__ */
# 990 "beam/erl_process_lock.h"
#endif /* #if !defined(ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__) 
	  && !defined(ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__) */
# 992 "beam/erl_process_lock.h"
# 2348 "beam/erl_process.h" 2

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 26 "beam/beam_load.h" 2

int erts_is_module_native(BeamInstr* code);
Eterm beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks,
			    Eterm module);

typedef struct gen_op_entry {
   char* name;
   int arity;
   int specific;
   int num_specific;
   int transform;
   int min_window;
} GenOpEntry;

extern GenOpEntry gen_opc[];

#ifdef NO_JUMP_TABLE 
#define BeamOp(Op) (Op)
#else
# 45 "beam/beam_load.h"
extern void** beam_ops;
#define BeamOp(Op) beam_ops[(Op)]
#endif
# 48 "beam/beam_load.h"


extern BeamInstr beam_debug_apply[];
extern BeamInstr* em_call_error_handler;
extern BeamInstr* em_apply_bif;
extern BeamInstr* em_call_nif;

/*
 * The following variables keep a sorted list of address ranges for
 * each module.  It allows us to quickly find a function given an
 * instruction pointer.
 */

/* Total code size in bytes */
extern Uint erts_total_code_size;
/*
 * Index into start of code chunks which contains additional information
 * about the loaded module.
 *
 * First number of functions.
 */

#define MI_NUM_FUNCTIONS     0

/*
 * The attributes retrieved by Mod:module_info(attributes).
 */

#define MI_ATTR_PTR          1
#define MI_ATTR_SIZE	     2
#define MI_ATTR_SIZE_ON_HEAP 3

/*
 * The compilation information retrieved by Mod:module_info(compile).
 */

#define MI_COMPILE_PTR          4
#define MI_COMPILE_SIZE         5
#define MI_COMPILE_SIZE_ON_HEAP 6

/*
 * Literal area (constant pool).
 */
#define MI_LITERALS_START	7
#define MI_LITERALS_END		8
#define MI_LITERALS_OFF_HEAP	9

/*
 * Pointer to the on_load function (or NULL if none).
 */
#define MI_ON_LOAD_FUNCTION_PTR 10

/*
 * Pointer to the line table (or NULL if none).
 */
#define MI_LINE_TABLE 11

/*
 * Pointer to the module MD5 sum (16 bytes)
 */
#define MI_MD5_PTR 12

/*
 * Start of function pointer table.  This table contains pointers to
 * all functions in the module plus an additional pointer just beyond
 * the end of the last function.
 *
 * The actual loaded code (for the first function) start just beyond
 * this table.
 */

#define MI_FUNCTIONS         13

/*
 * Layout of the line table.
 */

#define MI_LINE_FNAME_PTR 0
#define MI_LINE_LOC_TAB 1
#define MI_LINE_LOC_SIZE 2
#define MI_LINE_FUNC_TAB 3

#define LINE_INVALID_LOCATION (0)

/*
 * Macros for manipulating locations.
 */

#define IS_VALID_LOCATION(File, Line) \
    ((unsigned) (File) < 255 && (unsigned) (Line) < ((1 << 24) - 1))
#define MAKE_LOCATION(File, Line) (((File) << 24) | (Line))
#define LOC_FILE(Loc) ((Loc) >> 24)
#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))

#endif /* _BEAM_LOAD_H */
# 143 "beam/beam_load.h"
# 82 "beam/export.h" 2
#define ExportIsBuiltIn(EntryPtr) 			\
(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \
 ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif))

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Export*
erts_active_export_entry(Eterm m, Eterm f, unsigned int a)
{
    extern Export* erts_find_export_entry(Eterm m, Eterm f, unsigned a, ErtsCodeIndex);
    return erts_find_export_entry(m, f, a, erts_active_code_ix());
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 96 "beam/export.h"

#endif /* __EXPORT_H__ */
# 98 "beam/export.h"

# 34 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "module.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/global.h"
# 1 "beam/module.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __MODULE_H__
#define __MODULE_H__

#ifndef __INDEX_H__
#if 0 /* expanded by -frewrite-includes */
#include "index.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/module.h"
# 26 "beam/module.h"
#endif
# 27 "beam/module.h"

struct erl_module_instance {
    BeamInstr* code;
    int code_length;		/* Length of loaded code in bytes. */
    unsigned catches;
    struct erl_module_nif* nif;
    int num_breakpoints;
    int num_traced_exports;
};

typedef struct erl_module {
    IndexSlot slot;		/* Must be located at top of struct! */
    int module;			/* Atom index for module (not tagged). */

    struct erl_module_instance curr;
    struct erl_module_instance old; /* protected by "old_code" rwlock */
} Module; 

Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
Module* erts_put_module(Eterm mod);

void init_module_table(void);
void module_start_staging(void);
void module_end_staging(int commit);
void module_info(int, void *);

Module *module_code(int, ErtsCodeIndex);
int module_code_size(ErtsCodeIndex);
int module_table_sz(void);

ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex);
ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex);
ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex);
ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex);
#ifdef ERTS_ENABLE_LOCK_CHECK
int erts_is_old_code_rlocked(ErtsCodeIndex);
#endif
# 64 "beam/module.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

extern erts_smp_rwmtx_t the_old_code_rwlocks[ERTS_NUM_CODE_IX];

ERTS_GLB_INLINE void erts_rwlock_old_code(ErtsCodeIndex code_ix)
{
    erts_smp_rwmtx_rwlock(&the_old_code_rwlocks[code_ix]);
}
ERTS_GLB_INLINE void erts_rwunlock_old_code(ErtsCodeIndex code_ix)
{
    erts_smp_rwmtx_rwunlock(&the_old_code_rwlocks[code_ix]);
}
ERTS_GLB_INLINE void erts_rlock_old_code(ErtsCodeIndex code_ix)
{
    erts_smp_rwmtx_rlock(&the_old_code_rwlocks[code_ix]);
}
ERTS_GLB_INLINE void erts_runlock_old_code(ErtsCodeIndex code_ix)
{
    erts_smp_rwmtx_runlock(&the_old_code_rwlocks[code_ix]);
}

#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_is_old_code_rlocked(ErtsCodeIndex code_ix)
{
    return erts_smp_lc_rwmtx_is_rlocked(&the_old_code_rwlocks[code_ix]);
}
#endif
# 92 "beam/module.h"

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 94 "beam/module.h"


#endif /* !__MODULE_H__ */
# 97 "beam/module.h"
# 35 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "register.h"
#endif /* expanded by -frewrite-includes */
# 35 "beam/global.h"
# 1 "beam/register.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 1996-2012. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
** Registered processes
*/

#ifndef __REGPROC_H__
#define __REGPROC_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/register.h"
# 29 "beam/register.h"
#if 0 /* expanded by -frewrite-includes */
#include "hash.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/register.h"
# 30 "beam/register.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/register.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 31 "beam/register.h" 2
#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/register.h"
# 1 "beam/erl_port.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_PORT_TYPE__
#define ERL_PORT_TYPE__
typedef struct _erl_drv_port Port;
typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
#endif
# 26 "beam/erl_port.h"

#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__)
#define ERL_PORT_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_port_task.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port.h"
# 31 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_ptab.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_port.h"
# 32 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port.h"
# 33 "beam/erl_port.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_port.h"
# 34 "beam/erl_port.h"

#ifndef __WIN32__
#define ERTS_DEFAULT_MAX_PORTS (1 << 16)
#else
# 38 "beam/erl_port.h"
/* 
 *  Do not default to as many max ports on Windows
 *  as there are no os limits to stop system
 *  from running amok. If allowed to go too high
 *  windows rarely recovers from the errors and
 *  other OS processes can be effected. 
 */
#define ERTS_DEFAULT_MAX_PORTS (1 << 13)
#endif /* __WIN32__ */
# 47 "beam/erl_port.h"
#define ERTS_MIN_PORTS 1024

extern int erts_port_synchronous_ops;
extern int erts_port_schedule_all_ops;
extern int erts_port_parallelism;

typedef struct erts_driver_t_ erts_driver_t;

/*
 * It would have been preferred to use NULL as value of
 * ERTS_INVALID_ERL_DRV_PORT. That would, however, not be
 * backward compatible. In pre-R16 systems, 0 was a valid
 * port handle and -1 was used as invalid handle, so we
 * are stuck with it.
 */
#define ERTS_INVALID_ERL_DRV_PORT ((struct _erl_drv_port *) ((SWord) -1))
#ifdef DEBUG
/* Make sure we use this api, and do not cast directly */
#define ERTS_ErlDrvPort2Port(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((Port *) ((PH) - 4711)))
#define ERTS_Port2ErlDrvPort(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((ErlDrvPort) ((PH) + 4711)))
#else
# 74 "beam/erl_port.h"
#define ERTS_ErlDrvPort2Port(PH) ((Port *) (PH))
#define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH))
#endif
# 77 "beam/erl_port.h"

#define SMALL_IO_QUEUE 5   /* Number of fixed elements */

typedef struct {
    ErlDrvSizeT size;       /* total size in bytes */

    SysIOVec* v_start;
    SysIOVec* v_end;
    SysIOVec* v_head;
    SysIOVec* v_tail;
    SysIOVec  v_small[SMALL_IO_QUEUE];

    ErlDrvBinary** b_start;
    ErlDrvBinary** b_end;
    ErlDrvBinary** b_head;
    ErlDrvBinary** b_tail;
    ErlDrvBinary*  b_small[SMALL_IO_QUEUE];
} ErlIOQueue;

typedef struct line_buf {  /* Buffer used in line oriented I/O */
    ErlDrvSizeT bufsiz;      /* Size of character buffer */
    ErlDrvSizeT ovlen;       /* Length of overflow data */
    ErlDrvSizeT ovsiz;       /* Actual size of overflow buffer */
    char data[1];            /* Starting point of buffer data,
			      data[0] is a flag indicating an unprocess CR,
			      The rest is the overflow buffer. */
} LineBuf;

/*
 * Items part of erlang:port_info/1 result. Note am_registered_name
 * *need* to be first.
 */

#define ERTS_PORT_INFO_1_ITEMS				\
    {	am_registered_name,	/* Needs to be first */	\
	am_name,					\
	am_links,					\
	am_id,						\
	am_connected,					\
	am_input,					\
	am_output,					\
	am_os_pid }

/*
 * Port Specific Data.
 *
 * Only use PrtSD for very rarely used data.
 */

#define ERTS_PRTSD_SCHED_ID 0

#define ERTS_PRTSD_SIZE 1

typedef struct {
    void *data[ERTS_PRTSD_SIZE];
} ErtsPrtSD;

#ifdef ERTS_SMP
typedef struct ErtsXPortsList_ ErtsXPortsList;
#endif
# 137 "beam/erl_port.h"

/*
 * Port locking:
 *
 * Locking is done either driver specific or port specific. When
 * driver specific locking is used, all instances of the driver,
 * i.e. ports running the driver, share the same lock. When port
 * specific locking is used each instance have its own lock.
 *
 * Most fields in the Port structure are protected by the lock
 * referred to by the 'lock' field. This lock is shared between
 * all ports running the same driver when driver specific locking
 * is used.
 *
 * The 'sched' field is protected by the run queue lock that the
 * port currently is assigned to.
 *
 */

struct _erl_drv_port {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    ErtsPortTaskSched sched;
    ErtsPortTaskHandle timeout_task;
#ifdef ERTS_SMP
    erts_mtx_t *lock;
    ErtsXPortsList *xports;
    erts_smp_atomic_t run_queue;
#else
# 166 "beam/erl_port.h"
    erts_atomic32_t refc;
    int cleanup;
#endif
# 169 "beam/erl_port.h"
    erts_atomic_t connected;	/* A connected process */
    Eterm caller;		/* Current caller. */
    erts_smp_atomic_t data;	/* Data associated with port. */
    Uint bytes_in;		/* Number of bytes read */
    Uint bytes_out;		/* Number of bytes written */

    ErlIOQueue ioq;              /* driver accessible i/o queue */
    DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
    char *name;		         /* String used in the open */
    erts_driver_t* drv_ptr;
    UWord drv_data;
    SWord os_pid;                /* Child process ID */
    ErtsProcList *suspended;	 /* List of suspended processes. */
    LineBuf *linebuf;            /* Buffer to hold data not ready for
				    process to get (line oriented I/O)*/
    erts_atomic32_t state;	 /* Status and type flags */
    int control_flags;		 /* Flags for port_control()  */
    ErlDrvPDL port_data_lock;

    ErtsPrtSD *psd;		 /* Port specific data */
    int reds; /* Only used while executing driver callbacks */
};


void erts_init_port_data(Port *);
void erts_cleanup_port_data(Port *);
Uint erts_port_data_size(Port *);
ErlOffHeap *erts_port_data_offheap(Port *);

#define ERTS_PORT_GET_CONNECTED(PRT) \
    ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
    erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
    erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))


struct erl_drv_port_data_lock {
    erts_mtx_t mtx;
    erts_atomic_t refc;
    Port *prt;
};

ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsRunQueue *
erts_port_runq(Port *prt)
{
#ifdef ERTS_SMP
    ErtsRunQueue *rq1, *rq2;
    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
    if (!rq1)
	return NULL;
    while (1) {
	erts_smp_runq_lock(rq1);
	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
	if (rq1 == rq2)
	    return rq1;
	erts_smp_runq_unlock(rq1);
	rq1 = rq2;
	if (!rq1)
	    return NULL;
    }
#else
# 235 "beam/erl_port.h"
    return ERTS_RUNQ_IX(0);
#endif
# 237 "beam/erl_port.h"
}

#endif
# 240 "beam/erl_port.h"


ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix);
ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_prtsd_get(Port *prt, int ix)
{
    return prt->psd ? prt->psd->data[ix] : NULL;
}

ERTS_GLB_INLINE void *
erts_prtsd_set(Port *prt, int ix, void *data)
{
    if (prt->psd) {
	void *old = prt->psd->data[ix];
	prt->psd->data[ix] = data;
	return old;
    }
    else {
	prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
	prt->psd->data[ix] = data;
	return NULL;
    }
}

#endif
# 269 "beam/erl_port.h"

Eterm erts_request_io_bytes(Process *c_p);


/* port status flags */

#define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
/* Port have begun exiting */
#define ERTS_PORT_SFLG_EXITING		((Uint32) (1 <<  1))
/* Distribution port */
#define ERTS_PORT_SFLG_DISTRIBUTION	((Uint32) (1 <<  2))
#define ERTS_PORT_SFLG_BINARY_IO	((Uint32) (1 <<  3))
#define ERTS_PORT_SFLG_SOFT_EOF		((Uint32) (1 <<  4))
/* Flow control */
/* Port is closing (no i/o accepted) */
#define ERTS_PORT_SFLG_CLOSING		((Uint32) (1 <<  5))
/* Send a closed message when terminating */
#define ERTS_PORT_SFLG_SEND_CLOSED	((Uint32) (1 <<  6))
/* Line orinted io on port */  
#define ERTS_PORT_SFLG_LINEBUF_IO	((Uint32) (1 <<  7))
/* Immortal port (only certain system ports) */
#define ERTS_PORT_SFLG_FREE		((Uint32) (1 <<  8))
#define ERTS_PORT_SFLG_INITIALIZING	((Uint32) (1 <<  9))
/* Port uses port specific locking (opposed to driver specific locking) */
#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 10))
#define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 11))
/* Last port to terminate halts the emulator */
#define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 12))
#ifdef DEBUG
/* Only debug: make sure all flags aren't cleared unintentionally */
#define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
#endif
# 301 "beam/erl_port.h"

/* Combinations of port status flags */ 
#define ERTS_PORT_SFLGS_DEAD						\
  (ERTS_PORT_SFLG_FREE | ERTS_PORT_SFLG_INITIALIZING)
#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
  (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID)
#define ERTS_PORT_SFLGS_INVALID_LOOKUP					\
  (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
   | ERTS_PORT_SFLG_EXITING						\
   | ERTS_PORT_SFLG_CLOSING)
#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP				\
  (ERTS_PORT_SFLGS_INVALID_LOOKUP					\
   | ERTS_PORT_SFLG_DISTRIBUTION)

/*
 * Costs in reductions for some port operations.
 */
#define ERTS_PORT_REDS_EXECUTE		(CONTEXT_REDS/4)
#define ERTS_PORT_REDS_FREE		(CONTEXT_REDS/400)
#define ERTS_PORT_REDS_TIMEOUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_INPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_OUTPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EVENT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUTV	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUT	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EXIT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CONNECT		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_UNLINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_LINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_BADSIG		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_CONTROL		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CALL		(CONTEXT_REDS/50)
#define ERTS_PORT_REDS_INFO		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_TERMINATE	(CONTEXT_REDS/50)

void print_port_info(Port *, int, void *);
void erts_port_free(Port *);
#ifndef ERTS_SMP
void erts_port_cleanup(Port *);
#endif
# 341 "beam/erl_port.h"
void erts_fire_port_monitor(Port *prt, Eterm ref);
#ifdef ERTS_SMP
int erts_port_handle_xports(Port *);
#endif
# 345 "beam/erl_port.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_is_port_locked(Port *);
#endif
# 349 "beam/erl_port.h"

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);

ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt)
{
    erts_ptab_atmc_inc_refc(&prt->common);
}

ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt)
{
    int referred = erts_ptab_atmc_dec_test_refc(&prt->common);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc)
{
    int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
{
    return erts_ptab_atmc_read_refc(&prt->common);
}

ERTS_GLB_INLINE int
erts_smp_port_trylock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    return erts_mtx_trylock(prt->lock);
#else
# 393 "beam/erl_port.h"
    return 0;
#endif
# 395 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_lock(prt->lock);
#endif
# 405 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_unlock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_unlock(prt->lock);
#endif
# 415 "beam/erl_port.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 418 "beam/erl_port.h"


#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS)			\
    (!(PP)							\
     || (erts_atomic32_read_nob(&(PP)->state) & (FLGS))	\
     || (PP)->common.id != (ID))

/* port lookup */

#define INVALID_PORT(PP, ID) \
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP)

/* Invalidate trace port if anything suspicious, for instance
 * that the port is a distribution port or it is busy.
 */
#define INVALID_TRACER_PORT(PP, ID)					\
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)

#define ERTS_PORT_SCHED_ID(P, ID) \
  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))

extern const Port erts_invalid_port;
#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)

int erts_is_port_ioq_empty(Port *);
void erts_terminate_port(Port *);

#ifdef ERTS_SMP
Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
#endif
# 448 "beam/erl_port.h"

ERTS_GLB_INLINE Port *erts_pix2port(int);
ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm);
ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32);
ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
ERTS_GLB_INLINE void erts_port_release(Port *);
#ifdef ERTS_SMP
ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
#endif
# 459 "beam/erl_port.h"
ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int);
ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *);
ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort);
ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm);
ERTS_GLB_INLINE int erts_is_port_alive(Eterm);
ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm);
ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *);

#define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL)

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Port *erts_pix2port(int ix)
{
    Port *prt;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_port));
    prt = (Port *) erts_ptab_pix2intptr_nob(&erts_port, ix);
    return prt == ERTS_PORT_LOCK_BUSY ? NULL : prt;
}

ERTS_GLB_INLINE Port *
erts_port_lookup_raw(Eterm id)
{
    Port *prt;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));
    return prt && prt->common.id == id ? prt : NULL;
}

ERTS_GLB_INLINE Port *
erts_port_lookup(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt = erts_port_lookup_raw(id);
    return (!prt
	    ? NULL
	    : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state))
	       ? NULL
	       : prt));
}


ERTS_GLB_INLINE Port*
erts_id2port(Eterm id)
{
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

    erts_smp_port_lock(prt);
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) {
	erts_smp_port_unlock(prt);
	return NULL;
    }

    return prt;
}


ERTS_GLB_INLINE Port*
erts_id2port_sflgs(Eterm id,
		   Process *c_p, ErtsProcLocks c_p_locks,
		   Uint32 invalid_sflgs)
{
#ifdef ERTS_SMP
    int no_proc_locks = !c_p || !c_p_locks;
#endif
# 543 "beam/erl_port.h"
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

#ifdef ERTS_SMP
    if (no_proc_locks)
	erts_smp_port_lock(prt);
    else if (erts_smp_port_trylock(prt) == EBUSY) {
	/* Unlock process locks, and acquire locks in lock order... */
	erts_smp_proc_unlock(c_p, c_p_locks);
	erts_smp_port_lock(prt);
	erts_smp_proc_lock(c_p, c_p_locks);
    }
#endif
# 568 "beam/erl_port.h"
    state = erts_atomic32_read_nob(&prt->state);
    if (state & invalid_sflgs) {
#ifdef ERTS_SMP
	erts_smp_port_unlock(prt);
#endif
# 573 "beam/erl_port.h"
	return NULL;
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_port_release(Port *prt)
{
    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
#ifdef ERTS_SMP
    erts_smp_port_unlock(prt);
#else
# 587 "beam/erl_port.h"
    if (prt->cleanup) {
	prt->cleanup = 0;
	erts_port_cleanup(prt);
    }
#endif
# 592 "beam/erl_port.h"
}

#ifdef ERTS_SMP

/*
 * erts_thr_id2port_sflgs() and erts_thr_port_release() can
 * be used by unmanaged threads in the SMP case.
 */
ERTS_GLB_INLINE Port *
erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt;
    ErtsThrPrgrDelayHandle dhndl;

    if (is_not_internal_port(id))
	return NULL;

    dhndl = erts_thr_progress_unmanaged_delay();

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id) {
	erts_thr_progress_unmanaged_continue(dhndl);
	prt = NULL;
    }
    else {
	erts_aint32_t state;
	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
	    erts_port_inc_refc(prt);
	    erts_thr_progress_unmanaged_continue(dhndl);
	}

	erts_mtx_lock(prt->lock);
	state = erts_atomic32_read_nob(&prt->state);
	if (state & invalid_sflgs) {
	    erts_mtx_unlock(prt->lock);
	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
		erts_port_dec_refc(prt);
	    prt = NULL;
	}
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_thr_port_release(Port *prt)
{
    erts_mtx_unlock(prt->lock);
#ifdef ERTS_SMP
    if (!erts_thr_progress_is_managed_thread())
	erts_port_dec_refc(prt);
#endif
# 646 "beam/erl_port.h"
}

#endif
# 649 "beam/erl_port.h"

ERTS_GLB_INLINE Port *
erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    ASSERT(prt != NULL);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;

    if (lock_pdl && prt->port_data_lock)
	driver_pdl_lock(prt->port_data_lock);

#if ERTS_ENABLE_LOCK_CHECK
    if (!ERTS_IS_CRASH_DUMPING) {
	if (erts_lc_is_emu_thr()) {
	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	    ERTS_LC_ASSERT(!prt->port_data_lock
			   || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
	else {
	    ERTS_LC_ASSERT(prt->port_data_lock);
	    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
    }
#endif
# 674 "beam/erl_port.h"

    if (erts_atomic32_read_nob(&prt->state)
	& ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
	if (lock_pdl && prt->port_data_lock)
	    driver_pdl_unlock(prt->port_data_lock);
	return ERTS_INVALID_ERL_DRV_PORT;
    }
    return prt;
}

ERTS_GLB_INLINE Port *
erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    erts_aint32_t state;
    ASSERT(prt);
    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;
    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
		       || ERTS_IS_CRASH_DUMPING);
    /* 
     * This state check is only needed since a driver callback
     * might terminate the port, and then call back into the
     * emulator. Drivers should preferably have been forbidden
     * to call into the emulator after terminating the port,
     * but it has been like this for ages. Perhaps forbid this
     * in some future major release?
     */
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
	return ERTS_INVALID_ERL_DRV_PORT;
    if (statep)
	*statep = state;
    return prt;
}

ERTS_GLB_INLINE Eterm
erts_drvport2id(ErlDrvPort drvport)
{
    Port *prt = erts_drvport2port(drvport);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return am_undefined;
    else
	return prt->common.id;
}

ERTS_GLB_INLINE Uint32
erts_portid2status(Eterm id)
{
    Port *prt = erts_port_lookup_raw(id);
    if (prt)
	return (Uint32) erts_atomic32_read_acqb(&prt->state);
    else
	return ERTS_PORT_SFLG_INVALID;
}

ERTS_GLB_INLINE int
erts_is_port_alive(Eterm id)
{
    return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID
				       | ERTS_PORT_SFLGS_DEAD));
}

ERTS_GLB_INLINE int
erts_is_valid_tracer_port(Eterm id)
{
    return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
}

ERTS_GLB_INLINE int
erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
{
    int reds = 0;
    erts_aint32_t state;

    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));

    state = erts_atomic32_read_nob(&prt->state);
    if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) {
	reds += ERTS_PORT_REDS_TERMINATE;
	erts_terminate_port(prt);
	state = erts_atomic32_read_nob(&prt->state);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
    }

#ifdef ERTS_SMP
    if (prt->xports) {
	reds += erts_port_handle_xports(prt);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	ASSERT(!prt->xports);
    }
#endif
# 767 "beam/erl_port.h"

    if (statep)
	*statep = state;

    return reds;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 775 "beam/erl_port.h"

void erts_port_resume_procs(Port *);

struct binary;

#define ERTS_P2P_SIG_TYPE_BAD			0
#define ERTS_P2P_SIG_TYPE_OUTPUT		1
#define ERTS_P2P_SIG_TYPE_OUTPUTV		2
#define ERTS_P2P_SIG_TYPE_CONNECT		3
#define ERTS_P2P_SIG_TYPE_EXIT			4
#define ERTS_P2P_SIG_TYPE_CONTROL		5
#define ERTS_P2P_SIG_TYPE_CALL			6
#define ERTS_P2P_SIG_TYPE_INFO			7
#define ERTS_P2P_SIG_TYPE_LINK			8
#define ERTS_P2P_SIG_TYPE_UNLINK		9

#define ERTS_P2P_SIG_TYPE_BITS			4
#define ERTS_P2P_SIG_TYPE_MASK \
    ((1 << ERTS_P2P_SIG_TYPE_BITS) - 1)

#define ERTS_P2P_SIG_DATA_FLG(N) \
    (1 << (ERTS_P2P_SIG_TYPE_BITS + (N)))
#define ERTS_P2P_SIG_DATA_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG(0)
#define ERTS_P2P_SIG_DATA_FLG_REPLY		ERTS_P2P_SIG_DATA_FLG(1)
#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG(2)
#define ERTS_P2P_SIG_DATA_FLG_FORCE		ERTS_P2P_SIG_DATA_FLG(3)
#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT	ERTS_P2P_SIG_DATA_FLG(4)
#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK	ERTS_P2P_SIG_DATA_FLG(5)
#define ERTS_P2P_SIG_DATA_FLG_SCHED		ERTS_P2P_SIG_DATA_FLG(6)
#define ERTS_P2P_SIG_DATA_FLG_ASYNC		ERTS_P2P_SIG_DATA_FLG(7)

struct ErtsProc2PortSigData_ {
    int flags;
    Eterm caller;
    Uint32 ref[ERTS_MAX_REF_NUMBERS];
    union {
	struct {
	    Eterm from;
	    ErlIOVec *evp;
	    ErlDrvBinary *cbinp;
	} outputv;
	struct {
	    Eterm from;
	    char *bufp;
	    ErlDrvSizeT size;
	} output;
	struct {
	    Eterm from;
	    Eterm connected;
	} connect;
	struct {
	    Eterm from;
	    Eterm reason;
	    ErlHeapFragment *bp;
	} exit;
	struct {
	    struct binary *binp;
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} control;
	struct {
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} call;
	struct {
	    Eterm item;
	} info;
	struct {
	    Eterm port;
	    Eterm to;
	} link;
	struct {
	    Eterm from;
	} unlink;
    } u;
} ;

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp);
ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return !0;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return !0;
    default:				return 0;
    }
}

ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return sigdp->u.output.size;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return sigdp->u.outputv.evp->size;
    default:				return (ErlDrvSizeT) 0;
    }
}

#endif
# 882 "beam/erl_port.h"

#define ERTS_PROC2PORT_SIG_EXEC			0
#define ERTS_PROC2PORT_SIG_ABORT		1
#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND	2
#define ERTS_PROC2PORT_SIG_ABORT_CLOSED		3

typedef int (*ErtsProc2PortSigCallback)(Port *,
					erts_aint32_t,
					int,
					ErtsProc2PortSigData *);

typedef enum {
    ERTS_PORT_OP_BADARG,
    ERTS_PORT_OP_CALLER_EXIT,
    ERTS_PORT_OP_BUSY,
    ERTS_PORT_OP_BUSY_SCHEDULED,
    ERTS_PORT_OP_SCHEDULED,
    ERTS_PORT_OP_DROPPED,
    ERTS_PORT_OP_DONE
} ErtsPortOpResult;

ErtsPortOpResult
erts_schedule_proc2port_signal(Process *,
			       Port *,
			       Eterm,
			       Eterm *,
			       ErtsProc2PortSigData *,
			       int,
			       ErtsPortTaskHandle *,
			       ErtsProc2PortSigCallback);

int erts_deliver_port_exit(Port *, Eterm, Eterm, int);

/*
 * Port signal flags
 */
#define ERTS_PORT_SIG_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG_BANG_OP
#define ERTS_PORT_SIG_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG_NOSUSPEND
#define ERTS_PORT_SIG_FLG_FORCE			ERTS_P2P_SIG_DATA_FLG_FORCE
#define ERTS_PORT_SIG_FLG_BROKEN_LINK		ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK
#define ERTS_PORT_SIG_FLG_BAD_OUTPUT		ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT
#define ERTS_PORT_SIG_FLG_FORCE_SCHED		ERTS_P2P_SIG_DATA_FLG_SCHED
#define ERTS_PORT_SIG_FLG_ASYNC			ERTS_P2P_SIG_DATA_FLG_ASYNC
/* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */
#define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL	ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT

/*
 * Port ! {Owner, {command, Data}}
 * Port ! {Owner, {connect, NewOwner}}
 * Port ! {Owner, close}
 */
ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);

/*
 * Signals from processes to ports.
 */
ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);

#endif
# 948 "beam/erl_port.h"
# 33 "beam/register.h" 2
#undef ERL_PORT_GET_PORT_TYPE_ONLY__

typedef struct reg_proc
{
    HashBucket bucket;  /* MUST BE LOCATED AT TOP OF STRUCT!!! */
    Process *p;         /* The process registered (only one of this and
			   'pt' is non-NULL */
    Port *pt;		/* The port registered */
    Eterm name;         /* Atom name */
} RegProc;

int process_reg_size(void);
int process_reg_sz(void);
void init_register_table(void);
void register_info(int, void *);
int erts_register_name(Process *, Eterm, Eterm);
Eterm erts_whereis_name_to_id(Process *, Eterm);
void erts_whereis_name(Process *, ErtsProcLocks,
		       Eterm, Process**, ErtsProcLocks, int,
		       Port**);
Process *erts_whereis_process(Process *,
			      ErtsProcLocks,
			      Eterm,
			      ErtsProcLocks,
			      int);
int erts_unregister_name(Process *, ErtsProcLocks, Port *, Eterm);

#endif
# 61 "beam/register.h"
# 36 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_fun.h"
#endif /* expanded by -frewrite-includes */
# 36 "beam/global.h"
# 1 "beam/erl_fun.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2000-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERLFUNTABLE_H__
#define __ERLFUNTABLE_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_fun.h"
# 1 "beam/erl_smp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * SMP interface to ethread library.
 * This is essentially "sed s/erts_/erts_smp_/g < erl_threads.h > erl_smp.h",
 * plus changes to NOP operations when ERTS_SMP is disabled.
 * Author: Mikael Pettersson
 */
#ifndef ERL_SMP_H
#define ERL_SMP_H
#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/erl_smp.h"
# 29 "beam/erl_smp.h"

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_smp_mtx_lock(L) erts_smp_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_smp_mtx_trylock(L) erts_smp_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_smp_spin_lock(L) erts_smp_spin_lock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrlock(L) erts_smp_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rlock(L) erts_smp_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_tryrwlock(L) erts_smp_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_smp_rwmtx_rwlock(L) erts_smp_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_smp_read_lock(L) erts_smp_read_lock_x(L, __FILE__, __LINE__)
#define erts_smp_write_lock(L) erts_smp_write_lock_x(L, __FILE__, __LINE__)
#endif
# 41 "beam/erl_smp.h"


#ifdef ERTS_SMP
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER ERTS_THR_OPTS_DEFAULT_INITER
typedef erts_thr_opts_t erts_smp_thr_opts_t;
typedef erts_thr_init_data_t erts_smp_thr_init_data_t;
typedef erts_tid_t erts_smp_tid_t;
typedef erts_mtx_t erts_smp_mtx_t;
typedef erts_cnd_t erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER
#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED
#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_dw_atomic_t
#define erts_smp_atomic_t erts_atomic_t
#define erts_smp_atomic32_t erts_atomic32_t
#define erts_smp_atomic64_t erts_atomic64_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */

#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#else /* #ifdef ERTS_SMP */
# 75 "beam/erl_smp.h"

#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
typedef int erts_smp_thr_opts_t;
typedef int erts_smp_thr_init_data_t;
typedef int erts_smp_tid_t;
typedef int erts_smp_mtx_t;
typedef int erts_smp_cnd_t;
#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_SMP_RWMTX_TYPE_NORMAL 0
#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_SMP_RWMTX_LONG_LIVED 0
#define ERTS_SMP_RWMTX_SHORT_LIVED 0
#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
#define erts_smp_atomic_t erts_no_atomic_t
#define erts_smp_atomic32_t erts_no_atomic32_t
#define erts_smp_atomic64_t erts_no_atomic64_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
#else
# 105 "beam/erl_smp.h"
typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
# 108 "beam/erl_smp.h"

#define ERTS_SMP_MEMORY_BARRIER
#define ERTS_SMP_WRITE_MEMORY_BARRIER
#define ERTS_SMP_READ_MEMORY_BARRIER
#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#endif /* #ifdef ERTS_SMP */
# 115 "beam/erl_smp.h"

ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_smp_thr_create(erts_smp_tid_t *tid,
					 void * (*func)(void *),
					 void *arg,
					 erts_smp_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_smp_thr_detach(erts_smp_tid_t tid);
ERTS_GLB_INLINE void erts_smp_thr_exit(void *res);
ERTS_GLB_INLINE void erts_smp_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_smp_tid_t erts_smp_thr_self(void);
ERTS_GLB_INLINE int erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y);
#ifdef ERTS_HAVE_REC_MTX_INIT
#define ERTS_SMP_HAVE_REC_MTX_INIT 1
ERTS_GLB_INLINE void erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx);
#endif
# 131 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx,
					 char *name,
					 Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx,
						char *name,
						Eterm extra);
ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line);
#else
# 144 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_mtx_lock(erts_smp_mtx_t *mtx);
#endif
# 147 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_mtx_unlock(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_init(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_destroy(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd,
					   erts_smp_mtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd);
ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
					       erts_smp_rwmtx_opt_t *opt,
					       char *name,
					       Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
					     erts_smp_rwmtx_opt_t *opt,
					     char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 175 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
#endif
# 180 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
					      char *name,
					      Eterm extra);
ERTS_GLB_INLINE void erts_smp_spinlock_init(erts_smp_spinlock_t *lock,
					    char *name);
ERTS_GLB_INLINE void erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_spin_unlock(erts_smp_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line);
#else
# 194 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_spin_lock(erts_smp_spinlock_t *lock);
#endif
# 196 "beam/erl_smp.h"
ERTS_GLB_INLINE int erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock);
ERTS_GLB_INLINE void erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock,
					    char *name,
					    Eterm extra);
ERTS_GLB_INLINE void erts_smp_rwlock_init(erts_smp_rwlock_t *lock,
					  char *name);
ERTS_GLB_INLINE void erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_read_unlock(erts_smp_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line);
#else
# 208 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_read_lock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock);
#endif
# 211 "beam/erl_smp.h"
ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock);
ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp,
					     char *keyname);
ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key);
ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key);

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
					  const sigset_t *set,
					  sigset_t *oset);
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 227 "beam/erl_smp.h"

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of erl_threads.h for info on atomics.
 */

#ifdef ERTS_SMP

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob

#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb

#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb

#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb

#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb

#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb

#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb

#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_atomic_init_nob
#define erts_smp_atomic_set_nob erts_atomic_set_nob
#define erts_smp_atomic_read_nob erts_atomic_read_nob
#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
#define erts_smp_atomic_add_nob erts_atomic_add_nob
#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
#define erts_smp_atomic_read_bset_nob erts_atomic_read_bset_nob

#define erts_smp_atomic_init_mb erts_atomic_init_mb
#define erts_smp_atomic_set_mb erts_atomic_set_mb
#define erts_smp_atomic_read_mb erts_atomic_read_mb
#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
#define erts_smp_atomic_add_mb erts_atomic_add_mb
#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
#define erts_smp_atomic_read_bset_mb erts_atomic_read_bset_mb

#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
#define erts_smp_atomic_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_smp_atomic_init_relb erts_atomic_init_relb
#define erts_smp_atomic_set_relb erts_atomic_set_relb
#define erts_smp_atomic_read_relb erts_atomic_read_relb
#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
#define erts_smp_atomic_add_relb erts_atomic_add_relb
#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
#define erts_smp_atomic_read_bset_relb erts_atomic_read_bset_relb

#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
#define erts_smp_atomic_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
#define erts_smp_atomic_add_rb erts_atomic_add_rb
#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
#define erts_smp_atomic_read_bset_rb erts_atomic_read_bset_rb

#define erts_smp_atomic_init_wb erts_atomic_init_wb
#define erts_smp_atomic_set_wb erts_atomic_set_wb
#define erts_smp_atomic_read_wb erts_atomic_read_wb
#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
#define erts_smp_atomic_add_wb erts_atomic_add_wb
#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
#define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb

#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
#define erts_smp_atomic_read_dirty erts_atomic_read_dirty

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
#define erts_smp_atomic32_read_bset_nob erts_atomic32_read_bset_nob

#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
#define erts_smp_atomic32_read_bset_mb erts_atomic32_read_bset_mb

#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
#define erts_smp_atomic32_read_bset_acqb erts_atomic32_read_bset_acqb

#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
#define erts_smp_atomic32_read_bset_relb erts_atomic32_read_bset_relb

#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
#define erts_smp_atomic32_read_bset_ddrb erts_atomic32_read_bset_ddrb

#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
#define erts_smp_atomic32_read_bset_rb erts_atomic32_read_bset_rb

#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
#define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb

#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob

#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb

#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb

#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb

#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb

#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb

#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb

#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty

#else /* !ERTS_SMP */
# 606 "beam/erl_smp.h"

/* Double word size atomics */

#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_smp_atomic_init_nob erts_no_atomic_set
#define erts_smp_atomic_set_nob erts_no_atomic_set
#define erts_smp_atomic_read_nob erts_no_atomic_read
#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_smp_atomic_inc_nob erts_no_atomic_inc
#define erts_smp_atomic_dec_nob erts_no_atomic_dec
#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
#define erts_smp_atomic_add_nob erts_no_atomic_add
#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_smp_atomic_init_mb erts_no_atomic_set
#define erts_smp_atomic_set_mb erts_no_atomic_set
#define erts_smp_atomic_read_mb erts_no_atomic_read
#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_mb erts_no_atomic_inc
#define erts_smp_atomic_dec_mb erts_no_atomic_dec
#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
#define erts_smp_atomic_add_mb erts_no_atomic_add
#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_smp_atomic_init_acqb erts_no_atomic_set
#define erts_smp_atomic_set_acqb erts_no_atomic_set
#define erts_smp_atomic_read_acqb erts_no_atomic_read
#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_smp_atomic_add_acqb erts_no_atomic_add
#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_smp_atomic_init_relb erts_no_atomic_set
#define erts_smp_atomic_set_relb erts_no_atomic_set
#define erts_smp_atomic_read_relb erts_no_atomic_read
#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_relb erts_no_atomic_inc
#define erts_smp_atomic_dec_relb erts_no_atomic_dec
#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
#define erts_smp_atomic_add_relb erts_no_atomic_add
#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_smp_atomic_init_ddrb erts_no_atomic_set
#define erts_smp_atomic_set_ddrb erts_no_atomic_set
#define erts_smp_atomic_read_ddrb erts_no_atomic_read
#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_smp_atomic_add_ddrb erts_no_atomic_add
#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_rb erts_no_atomic_inc
#define erts_smp_atomic_dec_rb erts_no_atomic_dec
#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
#define erts_smp_atomic_add_rb erts_no_atomic_add
#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_smp_atomic_init_wb erts_no_atomic_set
#define erts_smp_atomic_set_wb erts_no_atomic_set
#define erts_smp_atomic_read_wb erts_no_atomic_read
#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_smp_atomic_inc_wb erts_no_atomic_inc
#define erts_smp_atomic_dec_wb erts_no_atomic_dec
#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
#define erts_smp_atomic_add_wb erts_no_atomic_add
#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_smp_atomic_set_dirty erts_no_atomic_set
#define erts_smp_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_smp_atomic32_init_nob erts_no_atomic32_set
#define erts_smp_atomic32_set_nob erts_no_atomic32_set
#define erts_smp_atomic32_read_nob erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_smp_atomic32_add_nob erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_mb erts_no_atomic32_set
#define erts_smp_atomic32_set_mb erts_no_atomic32_set
#define erts_smp_atomic32_read_mb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_mb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_relb erts_no_atomic32_set
#define erts_smp_atomic32_set_relb erts_no_atomic32_set
#define erts_smp_atomic32_read_relb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_relb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_rb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_smp_atomic32_init_wb erts_no_atomic32_set
#define erts_smp_atomic32_set_wb erts_no_atomic32_set
#define erts_smp_atomic32_read_wb erts_no_atomic32_read
#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_smp_atomic32_add_wb erts_no_atomic32_add
#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
#define erts_smp_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_smp_atomic64_init_nob erts_no_atomic64_set
#define erts_smp_atomic64_set_nob erts_no_atomic64_set
#define erts_smp_atomic64_read_nob erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_smp_atomic64_add_nob erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_mb erts_no_atomic64_set
#define erts_smp_atomic64_set_mb erts_no_atomic64_set
#define erts_smp_atomic64_read_mb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_mb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_relb erts_no_atomic64_set
#define erts_smp_atomic64_set_relb erts_no_atomic64_set
#define erts_smp_atomic64_read_relb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_relb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_rb erts_no_atomic64_set
#define erts_smp_atomic64_set_rb erts_no_atomic64_set
#define erts_smp_atomic64_read_rb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_rb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_smp_atomic64_init_wb erts_no_atomic64_set
#define erts_smp_atomic64_set_wb erts_no_atomic64_set
#define erts_smp_atomic64_read_wb erts_no_atomic64_read
#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_smp_atomic64_add_wb erts_no_atomic64_add
#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
#define erts_smp_atomic64_read_dirty erts_no_atomic64_read

#endif /* !ERTS_SMP */
# 978 "beam/erl_smp.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_thr_init(erts_smp_thr_init_data_t *id)
{
#ifdef ERTS_SMP
    erts_thr_init(id);
#endif
# 987 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_create(erts_smp_tid_t *tid, void * (*func)(void *), void *arg,
		    erts_smp_thr_opts_t *opts)
{
#ifdef ERTS_SMP
    erts_thr_create(tid, func, arg, opts);
#endif
# 996 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_join(erts_smp_tid_t tid, void **thr_res)
{
#ifdef ERTS_SMP
    erts_thr_join(tid, thr_res);
#endif
# 1004 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_detach(erts_smp_tid_t tid)
{
#ifdef ERTS_SMP
    erts_thr_detach(tid);
#endif
# 1013 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_thr_exit(void *res)
{
#ifdef ERTS_SMP
    erts_thr_exit(res);
#endif
# 1022 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_install_exit_handler(void (*exit_handler)(void))
{
#ifdef ERTS_SMP
    erts_thr_install_exit_handler(exit_handler);
#endif
# 1030 "beam/erl_smp.h"
}

ERTS_GLB_INLINE erts_smp_tid_t
erts_smp_thr_self(void)
{
#ifdef ERTS_SMP
    return erts_thr_self();
#else
# 1038 "beam/erl_smp.h"
    return 0;
#endif
# 1040 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
erts_smp_equal_tids(erts_smp_tid_t x, erts_smp_tid_t y)
{
#ifdef ERTS_SMP
    return erts_equal_tids(x, y);
#else
# 1049 "beam/erl_smp.h"
    return 1;
#endif
# 1051 "beam/erl_smp.h"
}


#ifdef ERTS_HAVE_REC_MTX_INIT
ERTS_GLB_INLINE void
erts_smp_rec_mtx_init(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_rec_mtx_init(mtx);
#endif
# 1061 "beam/erl_smp.h"
}
#endif
# 1063 "beam/erl_smp.h"

ERTS_GLB_INLINE void
erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_x(mtx, name, extra, 1);
#endif
# 1070 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked_x(mtx, name, extra, 1);
#endif
# 1078 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init(mtx, name);
#endif
# 1086 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name)
{
#ifdef ERTS_SMP
    erts_mtx_init_locked(mtx, name);
#endif
# 1094 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_destroy(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(mtx);
#endif
# 1102 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_trylock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1108 "beam/erl_smp.h"
erts_smp_mtx_trylock(erts_smp_mtx_t *mtx)
#endif
# 1110 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_mtx_trylock_x(mtx,file,line);
#elif defined(ERTS_SMP)
# 1114 "beam/erl_smp.h"
    return erts_mtx_trylock(mtx);
#else
# 1116 "beam/erl_smp.h"
    return 0;
#endif
# 1118 "beam/erl_smp.h"

}


ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, unsigned int line)
#else
# 1126 "beam/erl_smp.h"
erts_smp_mtx_lock(erts_smp_mtx_t *mtx)
#endif
# 1128 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_mtx_lock_x(mtx, file, line);
#elif defined(ERTS_SMP)
# 1132 "beam/erl_smp.h"
    erts_mtx_lock(mtx);
#endif
# 1134 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_mtx_unlock(erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(mtx);
#endif
# 1142 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_mtx_is_locked(erts_smp_mtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(mtx);
#else
# 1150 "beam/erl_smp.h"
    return 0;
#endif
# 1152 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_init(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_init(cnd);
#endif
# 1160 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_destroy(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_destroy(cnd);
#endif
# 1168 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx)
{
#ifdef ERTS_SMP
    erts_cnd_wait(cnd, mtx);
#endif
# 1176 "beam/erl_smp.h"
}

/*
 * IMPORTANT note about erts_smp_cnd_signal() and erts_smp_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_smp_cnd_signal()/erts_smp_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_smp_cnd_signal(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_signal(cnd);
#endif
# 1194 "beam/erl_smp.h"
}


ERTS_GLB_INLINE void
erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd)
{
#ifdef ERTS_SMP
    erts_cnd_broadcast(cnd);
#endif
# 1203 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_set_reader_group(int no)
{
#ifdef ERTS_SMP
    erts_rwmtx_set_reader_group(no);
#endif
# 1211 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx,
			  erts_smp_rwmtx_opt_t *opt,
			  char *name,
			  Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt_x(rwmtx, opt, name, extra);
#endif
# 1222 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_x(rwmtx, name, extra);
#endif
# 1230 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx,
			erts_smp_rwmtx_opt_t *opt,
			char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init_opt(rwmtx, opt, name);
#endif
# 1240 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name)
{
#ifdef ERTS_SMP
    erts_rwmtx_init(rwmtx, name);
#endif
# 1248 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_destroy(rwmtx);
#endif
# 1256 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1262 "beam/erl_smp.h"
erts_smp_rwmtx_tryrlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1264 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1268 "beam/erl_smp.h"
    return erts_rwmtx_tryrlock(rwmtx);
#else
# 1270 "beam/erl_smp.h"
    return 0;
#endif
# 1272 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1278 "beam/erl_smp.h"
erts_smp_rwmtx_rlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1280 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1284 "beam/erl_smp.h"
    erts_rwmtx_rlock(rwmtx);
#endif
# 1286 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_runlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_runlock(rwmtx);
#endif
# 1294 "beam/erl_smp.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_tryrwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1301 "beam/erl_smp.h"
erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1303 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    return erts_rwmtx_tryrwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1307 "beam/erl_smp.h"
    return erts_rwmtx_tryrwlock(rwmtx);
#else
# 1309 "beam/erl_smp.h"
    return 0;
#endif
# 1311 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_rwmtx_rwlock_x(erts_smp_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 1317 "beam/erl_smp.h"
erts_smp_rwmtx_rwlock(erts_smp_rwmtx_t *rwmtx)
#endif
# 1319 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_rwmtx_rwlock_x(rwmtx, file, line);
#elif defined(ERTS_SMP)
# 1323 "beam/erl_smp.h"
    erts_rwmtx_rwlock(rwmtx);
#endif
# 1325 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx)
{
#ifdef ERTS_SMP
    erts_rwmtx_rwunlock(rwmtx);
#endif
# 1333 "beam/erl_smp.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_smp_rwmtx_trywlock(erts_smp_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wlock(erts_smp_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_smp_rwmtx_wunlock(erts_smp_rwmtx_t *rwmtx)
{

}

#endif
# 1360 "beam/erl_smp.h"

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rlocked(mtx);
#else
# 1367 "beam/erl_smp.h"
    return 0;
#endif
# 1369 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwmtx_is_rwlocked(mtx);
#else
# 1377 "beam/erl_smp.h"
    return 0;
#endif
# 1379 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_spinlock_init_x(lock, name, extra);
#else
# 1387 "beam/erl_smp.h"
    (void)lock;
#endif
# 1389 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_init(erts_smp_spinlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_spinlock_init(lock, name);
#else
# 1397 "beam/erl_smp.h"
    (void)lock;
#endif
# 1399 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spinlock_destroy(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spinlock_destroy(lock);
#else
# 1407 "beam/erl_smp.h"
    (void)lock;
#endif
# 1409 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_spin_unlock(erts_smp_spinlock_t *lock)
{
#ifdef ERTS_SMP
    erts_spin_unlock(lock);
#else
# 1417 "beam/erl_smp.h"
    (void)lock;
#endif
# 1419 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_spin_lock_x(erts_smp_spinlock_t *lock, char *file, unsigned int line)
#else
# 1425 "beam/erl_smp.h"
erts_smp_spin_lock(erts_smp_spinlock_t *lock)
#endif
# 1427 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_spin_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1431 "beam/erl_smp.h"
    erts_spin_lock(lock);
#else
# 1433 "beam/erl_smp.h"
    (void)lock;
#endif
# 1435 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_spinlock_is_locked(erts_smp_spinlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_spinlock_is_locked(lock);
#else
# 1443 "beam/erl_smp.h"
    return 0;
#endif
# 1445 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init_x(erts_smp_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
    erts_rwlock_init_x(lock, name, extra);
#else
# 1453 "beam/erl_smp.h"
    (void)lock;
#endif
# 1455 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_init(erts_smp_rwlock_t *lock, char *name)
{
#ifdef ERTS_SMP
    erts_rwlock_init(lock, name);
#else
# 1463 "beam/erl_smp.h"
    (void)lock;
#endif
# 1465 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_rwlock_destroy(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_rwlock_destroy(lock);
#else
# 1473 "beam/erl_smp.h"
    (void)lock;
#endif
# 1475 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_read_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_read_unlock(lock);
#else
# 1483 "beam/erl_smp.h"
    (void)lock;
#endif
# 1485 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_read_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1491 "beam/erl_smp.h"
erts_smp_read_lock(erts_smp_rwlock_t *lock)
#endif 
# 1493 "beam/erl_smp.h"
{
#if defined(ERTS_ENABLE_LOCK_POSITION) && defined(ERTS_SMP)
    erts_read_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1497 "beam/erl_smp.h"
    erts_read_lock(lock);
#else
# 1499 "beam/erl_smp.h"
    (void)lock;
#endif
# 1501 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_write_unlock(erts_smp_rwlock_t *lock)
{
#ifdef ERTS_SMP
    erts_write_unlock(lock);
#else
# 1509 "beam/erl_smp.h"
    (void)lock;
#endif
# 1511 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_smp_write_lock_x(erts_smp_rwlock_t *lock, char *file, unsigned int line)
#else
# 1517 "beam/erl_smp.h"
erts_smp_write_lock(erts_smp_rwlock_t *lock)
#endif 
# 1519 "beam/erl_smp.h"
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_POSITION)
    erts_write_lock_x(lock, file, line);
#elif defined(ERTS_SMP)
# 1523 "beam/erl_smp.h"
    erts_write_lock(lock);
#else
# 1525 "beam/erl_smp.h"
    (void)lock;
#endif
# 1527 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rlocked(lock);
#else
# 1535 "beam/erl_smp.h"
    return 0;
#endif
# 1537 "beam/erl_smp.h"
}

ERTS_GLB_INLINE int
erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_rwlock_is_rwlocked(lock);
#else
# 1545 "beam/erl_smp.h"
    return 0;
#endif
# 1547 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp, char* keyname)
{
#ifdef ERTS_SMP
    erts_tsd_key_create(keyp,keyname);
#endif
# 1555 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_key_delete(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    erts_tsd_key_delete(key);
#endif
# 1563 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value)
{
#ifdef ERTS_SMP
    erts_tsd_set(key, value);
#endif
# 1571 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void *
erts_smp_tsd_get(erts_smp_tsd_key_t key)
{
#ifdef ERTS_SMP
    return erts_tsd_get(key);
#else
# 1579 "beam/erl_smp.h"
    return NULL;
#endif
# 1581 "beam/erl_smp.h"
}

#ifdef ERTS_THR_HAVE_SIG_FUNCS
#define ERTS_SMP_THR_HAVE_SIG_FUNCS 1

ERTS_GLB_INLINE void
erts_smp_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef ERTS_SMP
    erts_thr_sigmask(how, set, oset);
#endif
# 1592 "beam/erl_smp.h"
}

ERTS_GLB_INLINE void
erts_smp_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef ERTS_SMP
    erts_thr_sigwait(set, sig);
#endif
# 1600 "beam/erl_smp.h"
}

#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
# 1603 "beam/erl_smp.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1605 "beam/erl_smp.h"

#endif /* ERL_SMP_H */
# 1607 "beam/erl_smp.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_smp_atomic_init
#undef erts_smp_atomic_set
#undef erts_smp_atomic_read
#undef erts_smp_atomic_inctest
#undef erts_smp_atomic_dectest
#undef erts_smp_atomic_inc
#undef erts_smp_atomic_dec
#undef erts_smp_atomic_addtest
#undef erts_smp_atomic_add
#undef erts_smp_atomic_xchg
#undef erts_smp_atomic_cmpxchg
#undef erts_smp_atomic_bor
#undef erts_smp_atomic_band

#undef erts_smp_atomic32_init
#undef erts_smp_atomic32_set
#undef erts_smp_atomic32_read
#undef erts_smp_atomic32_inctest
#undef erts_smp_atomic32_dectest
#undef erts_smp_atomic32_inc
#undef erts_smp_atomic32_dec
#undef erts_smp_atomic32_addtest
#undef erts_smp_atomic32_add
#undef erts_smp_atomic32_xchg
#undef erts_smp_atomic32_cmpxchg
#undef erts_smp_atomic32_bor
#undef erts_smp_atomic32_band

#endif
# 1641 "beam/erl_smp.h"
# 25 "beam/erl_fun.h" 2

/*
 * Fun entry.
 */

typedef struct erl_fun_entry {
    HashBucket bucket;		/* MUST BE LOCATED AT TOP OF STRUCT!!! */

    byte uniq[16];		/* MD5 for module. */
    int index;			/* New style index. */
    int old_uniq;		/* Unique number (old_style) */
    int old_index;		/* Old style index */
    BeamInstr* address;		/* Pointer to code for fun */

#ifdef HIPE
    UWord* native_address;	/* Native entry code for fun. */
#endif
# 42 "beam/erl_fun.h"

    Uint arity;			/* The arity of the fun. */
    Eterm module;		/* Tagged atom for module. */
    erts_refc_t refc;		/* Reference count: One for code + one for each
				   fun object in each process. */
} ErlFunEntry;

/*
 * This structure represents a 'fun' (lambda). It is stored on
 * process heaps. It has variable size depending on the size
 * of the environment.
 */

typedef struct erl_fun_thing {
    Eterm thing_word;		/* Subtag FUN_SUBTAG. */
    ErlFunEntry* fe;		/* Pointer to fun entry. */
    struct erl_off_heap_header* next;
#ifdef HIPE
    UWord* native_address;	/* Native code for the fun. */
#endif
# 62 "beam/erl_fun.h"
    Uint arity;			/* The arity of the fun. */
    Uint num_free;		/* Number of free variables (in env). */
  /* -- The following may be compound Erlang terms ---------------------- */
    Eterm creator;		/* Pid of creator process (contains node). */
    Eterm env[1];		/* Environment (free variables). */
} ErlFunThing;

/* ERL_FUN_SIZE does _not_ include space for the environment */
#define ERL_FUN_SIZE ((sizeof(ErlFunThing)/sizeof(Eterm))-1)

void erts_init_fun_table(void);
void erts_fun_info(int, void *);
int erts_fun_table_sz(void);

ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index);
ErlFunEntry* erts_get_fun_entry(Eterm mod, int uniq, int index);

ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index,
				byte* uniq, int index, int arity);

void erts_erase_fun_entry(ErlFunEntry* fe);
void erts_cleanup_funs(ErlFunThing* funp);
void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end);
void erts_dump_fun_entries(int, void *);

#endif
# 88 "beam/erl_fun.h"
# 37 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 37 "beam/global.h"
# 38 "beam/global.h"
#if 0 /* expanded by -frewrite-includes */
#include "benchmark.h"
#endif /* expanded by -frewrite-includes */
# 38 "beam/global.h"
# 1 "beam/benchmark.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2002-2012. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

#ifndef __BENCHMARK_H__
#define __BENCHMARK_H__

/* The define __BENCHMARK__ is the master switch to turn on and off
 * benchmarking. This will enable the benchmark-BIFs in hipe_bif1.c.
 * Documentation for the BIFs is in hipe_bif1.c, and that is where you
 * will find the information about how to accually get some data out
 * from these timers and counters.
 */
/* #define __BENCHMARK__ */

#ifdef __BENCHMARK__
/*
 * The defines below enables different parts of the benchmaring.
 * Counters and timers that are disabled, always report zero in
 * the BIFs.
 */

/* BM_TIMERS keeps track of the time spent in diferent parts of the
 * system. It only measures accual active time, not time spent in idle
 * mode. Currently, the Solaris hrtime_t will be used.
 * To add new timers look below.
 */
#define BM_TIMERS

/* BM_COUNTERS count all kinds of events that occurs in the system.
 * Among other things it counts the number of messages, then number of
 * garbage collections, the number of processes spawned etc.
 * To add new counters look below.
 */
#define BM_COUNTERS

/* BM_MESSAGE_SIZES keeps a log of the size of all messages sent in
 * the system. This introduce an overhead in time for the shared heap
 * system since all message sizes have to be calculated at send.
 */
/* #define BM_MESSAGE_SIZES */

/* BM_HEAP_SIZES goes through all processes at garbage collection time
 * to sum their allocated and used heap sizes. In anything else than a
 * shared heap system, this will cost.
 */
/* #define BM_HEAP_SIZES */

/* BM_STATISTICS saves an entry in the file BM_STATISTICS_FILE. This
 * is done for each erlang node at exit time.
 */
/* #define BM_STATISTICS */

#endif /* __BENCHMARK__ */
# 71 "beam/benchmark.h"


#ifdef BM_STATISTICS
#  define BM_STATISTICS_FILE "/tmp/erlang_statistics.joppe.log"
#endif /* BM_STATISTICS */
# 76 "beam/benchmark.h"


/************ There are no more settings below this line *************/

/*
 * Maintenance and how to add new stuff is documented by the code
 * below ;-)
 */

#ifdef BM_COUNTERS
/*********************************************************************
 * To add new counters:
 *
 * Add the variable here AND in benchmark.c. Use the macro
 * BM_COUNT(var) in the code where you want to increase it.
 *
 */
extern unsigned long long processes_busy;
extern unsigned long long processes_spawned;
extern unsigned long long messages_sent;
extern unsigned long long messages_copied;
extern unsigned long long messages_ego;
extern unsigned long long minor_gc;
extern unsigned long long major_gc;

#define BM_COUNT(var) (var)++;

#define BM_EGO_COUNT(send,rec) {     \
    if ((send) == (rec))             \
        BM_COUNT(messages_ego);      }

#define BM_LAZY_COPY_START long long gcs = minor_global_gc + major_global_gc;
#define BM_LAZY_COPY_STOP { gcs = (minor_global_gc + major_global_gc) - gcs; \
    if (gcs > gc_in_copy) gc_in_copy = gcs; }

#else /* !BM_COUNTERS */
# 112 "beam/benchmark.h"
#  define BM_COUNT(var)
#  define BM_EGO_COUNT(send,rec)
#  define BM_LAZY_COPY_START
#  define BM_LAZY_COPY_STOP
#endif /* BM_COUNTERS */
# 117 "beam/benchmark.h"


#ifdef BM_TIMERS
/*********************************************************************
 * To add new timers:
 *
 * Add the variable below using the form extern BM_TIMER_T blah_time.
 * Also add them in benchmark.c using the macro NEW_TIMER(blah).  Use
 * the macro BM_SWAP_TIMER(from,blah) ... BM_SWAP_TIMER(blah,to) to
 * start and stop the new timer. Note, that you have to know what
 * timer is running at the place where you want to insert your new
 * timer to be able to stop and start (from,to) it.
 *
 * You can use the macros BM_STOP_TIMER(blah) and BM_START_TIMER(blah)
 * around code that should not be timed at all. As above, you have to
 * know what timer to start and stop. The system timer is running at
 * most places in the emulator. Only the garbage collector and the
 * message sending has its own timers at the moment.
 *
 * The timer_time used when stopping timers is the time it takes to
 * start and stop the timers, calculated in init_benchmarking(). If it
 * is not there, the time it takes to do this will accually be
 * substantial compared to some small times in the system we want to
 * meassure (send time in shared heap for instance).
 */

/* (Assuming Solaris) */

#define BM_TIMER_T ErtsMonotonicTime
#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time())
#define BM_STOP_TIMER(t) do {                                        \
    BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time;  \
    t##_time += (tmp > 0 ? tmp : 0);                                 \
} while(0)

#define BM_TIME_PRINTER(str,time) do {                               \
    int min,sec,milli,micro;                                         \
    BM_TIMER_T tmp;                                                  \
    tmp = (time) / 1000;                                             \
    micro = tmp % 1000;                                              \
    tmp /= 1000;                                                     \
    milli = tmp % 1000;                                              \
    tmp /= 1000;                                                     \
    sec = tmp % 60;                                                  \
    min = tmp / 60;                                                  \
    erts_fprintf(file,str": %d:%02d.%03d %03d\n",min,sec,milli,micro);  \
} while(0)

extern BM_TIMER_T system_clock;

extern BM_TIMER_T timer_time;
extern BM_TIMER_T system_time;
extern BM_TIMER_T gc_time;
extern BM_TIMER_T minor_gc_time;
extern BM_TIMER_T major_gc_time;
extern BM_TIMER_T minor_global_gc_time;
extern BM_TIMER_T major_global_gc_time;
extern BM_TIMER_T send_time;
extern BM_TIMER_T copy_time;
extern BM_TIMER_T size_time;
extern BM_TIMER_T max_minor_time;
extern BM_TIMER_T max_major_time;
extern BM_TIMER_T max_global_minor_time;
extern BM_TIMER_T max_global_major_time;
extern BM_TIMER_T misc0_time;
extern BM_TIMER_T misc1_time;
extern BM_TIMER_T misc2_time;

#define MAX_PAUSE_TIME 500000
extern unsigned long local_pause_times[MAX_PAUSE_TIME];
extern unsigned long pause_times[MAX_PAUSE_TIME];
extern unsigned long pause_times_old[MAX_PAUSE_TIME];

#define MMU_INTERVAL 5  /* milli seconds */
extern BM_TIMER_T mmu_counter;
extern BM_TIMER_T mmu;

#define BM_NEW_TIMER(t) BM_TIMER_T t##_time = 0;
#define BM_RESET_TIMER(t) t##_time = 0;
#define BM_SWAP_TIMER(t1,t2) do { BM_STOP_TIMER(t1); BM_START_TIMER(t2); } while(0)
#define BM_MMU_INIT() do {                                              \
    BM_TIMER_T gc = gc_time;                                            \
    while (gc > 0) {                                                    \
        if (gc > MMU_INTERVAL) {                                        \
            gc -= MMU_INTERVAL - mmu_counter;                           \
            erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100));      \
            mmu_counter = 0; mmu = 0;                                   \
        } else {                                                        \
            mmu_counter += gc;                                          \
            if (mmu_counter >= MMU_INTERVAL) {                          \
                mmu_counter -= MMU_INTERVAL;                            \
                erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100));  \
                mmu = 0;                                                \
            }                                                           \
            gc = 0;                                                     \
        }                                                               \
    }                                                                   \
    BM_RESET_TIMER(system);                                             \
    BM_RESET_TIMER(send);                                               \
    BM_RESET_TIMER(copy);                                               \
    BM_RESET_TIMER(size);                                               \
} while(0)

#define BM_MMU_READ() do {                                              \
    BM_TIMER_T mut = system_time + send_time + copy_time + size_time;   \
    while (mut > 0) {                                                   \
        if (mut > MMU_INTERVAL) {                                       \
            BM_TIMER_T tmp = MMU_INTERVAL - mmu_counter;                \
            mmu += tmp; mut -= tmp;                                     \
            erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100));      \
            mmu_counter = 0; mmu = 0;                                   \
        } else {                                                        \
            mmu_counter += mut; mmu += mut;                             \
            if (mmu_counter >= MMU_INTERVAL) {                          \
                mmu_counter -= MMU_INTERVAL;                            \
                mmu -= mmu_counter;                                     \
                erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100));  \
                mmu = mmu_counter;                                      \
            }                                                           \
            mut = 0;                                                    \
        }                                                               \
    }                                                                   \
} while(0)

#else /* !BM_TIMERS */
# 242 "beam/benchmark.h"
#  define BM_NEW_TIMER(t)
#  define BM_START_TIMER(t)
#  define BM_STOP_TIMER(t)
#  define BM_RESET_TIMER(t)
#  define BM_SWAP_TIMER(t1,t2)
#  define BM_TIME_PRINTER(str,time)
#  define BM_MMU_INIT()
#  define BM_MMU_READ()
#endif /* BM_TIMERS */
# 251 "beam/benchmark.h"

#ifdef BM_HEAP_SIZES
extern unsigned long long max_used_heap;
extern unsigned long long max_allocated_heap;
extern unsigned long long max_used_global_heap;
extern unsigned long long max_allocated_global_heap;
#endif /* BM_HEAP_SIZES */
# 258 "beam/benchmark.h"

#ifdef BM_MESSAGE_SIZES
extern unsigned long long words_sent;
extern unsigned long long words_copied;
extern unsigned long long words_prealloc;
extern unsigned long long message_sizes[1000];

#define BM_MESSAGE_COPIED(size) { \
    words_copied += size;         \
    BM_COUNT(messages_copied);    }

#define BM_PREALLOC_DATA(size) { \
    words_prealloc += size;      }

#define BM_MESSAGE(mess,send,rec) {  \
    Uint msize = size_object(mess);  \
    words_sent += msize;             \
    if (msize < 1000)                \
        message_sizes[msize]++;      \
    else                             \
        message_sizes[999]++;        \
    BM_EGO_COUNT(send,rec);          \
    BM_COUNT(messages_sent);         }

#else /* !BM_MESSAGE_SIZES */
# 283 "beam/benchmark.h"

#define BM_MESSAGE_COPIED(size) BM_COUNT(messages_copied);
#define BM_PREALLOC_DATA(size)
#define BM_MESSAGE(mess,send,rec) {  \
    BM_EGO_COUNT(send,rec);          \
    BM_COUNT(messages_sent);         }

#endif /* BM_MESSAGE_SIZES */
# 291 "beam/benchmark.h"

void init_benchmarking(void);
void save_statistics(void);

#endif /* _BENCHMARK_H_ */
# 296 "beam/benchmark.h"
# 39 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 39 "beam/global.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 40 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_sys_driver.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/global.h"
# 41 "beam/global.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_debug.h"
#endif /* expanded by -frewrite-includes */
# 41 "beam/global.h"
# 1 "beam/erl_debug.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2004-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
#ifndef _ERL_DEBUG_H_
#define _ERL_DEBUG_H_
#ifdef DEBUG

#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_debug.h"
# 25 "beam/erl_debug.h"

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_debug.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/erl_debug.h"
# 28 "beam/erl_debug.h"
#endif
# 29 "beam/erl_debug.h"

/* Heap areas will be filled with this value when they are deallocated
 * after a garbage collection. This value used to be 0xff, but that is
 * an immediate and might not crash the system if it is encountered.
 * The value is now 0x01, the cons of death.
 */
#define DEBUG_BAD_BYTE 0x01
#define DEBUG_BAD_WORD 0x01010101

/*
 * VERBOSE. Use the -v option to enable the different categories.
 */
#define VERBOSE(flag, format) (flag & verbose ? erts_printf format : 0)

#define DEBUG_DEFAULT      0x0000    /* No flags are set per default         */
#define DEBUG_SYSTEM       0x0001    /* Misc system info at startup and end  */
#define DEBUG_PRIVATE_GC   0x0002    /* GC of private heaps                  */
#define DEBUG_ALLOCATION   0x0004    /* HAlloc. To find holes in the heap    */
#define DEBUG_MESSAGES     0x0008    /* Message passing                      */
#define DEBUG_THREADS      0x0010    /* Thread-related stuff                 */
#define DEBUG_PROCESSES    0x0020    /* Process creation and removal         */
#define DEBUG_MEMORY       0x0040    /* Display results of memory checks     */

extern Uint32 verbose;

void upp(byte*, size_t);
void pat(Eterm);
void pinfo(void);
void pp(Process*);
void ppi(Eterm);
void pba(Process*, int);
void td(Eterm);
void ps(Process*, Eterm*);

#undef ERTS_OFFHEAP_DEBUG
#define ERTS_OFFHEAP_DEBUG

#else /* Non-debug mode */
# 67 "beam/erl_debug.h"

#define VERBOSE(flag,format)

#endif /* DEBUG */
# 71 "beam/erl_debug.h"

#ifdef ERTS_OFFHEAP_DEBUG
#define ERTS_CHK_OFFHEAP(P) erts_check_off_heap((P))
#define ERTS_CHK_OFFHEAP2(P, HT) erts_check_off_heap2((P), (HT))
void erts_check_off_heap(Process *);
void erts_check_off_heap2(Process *, Eterm *);
#else
# 78 "beam/erl_debug.h"
#define ERTS_CHK_OFFHEAP(P)
#define ERTS_CHK_OFFHEAP2(P, HT)
#endif
# 81 "beam/erl_debug.h"

/*
 * These functions can be handy when developing, and perhaps useful
 * even outside debugging.
 */
extern void erts_check_off_heap(Process *p);
extern void erts_check_stack(Process *p);
extern void erts_check_heap(Process *p);
extern void erts_check_memory(Process *p, Eterm *start, Eterm *end);
extern void verify_process(Process *p);
extern void print_tagged_memory(Eterm *start, Eterm *end);
extern void print_untagged_memory(Eterm *start, Eterm *end);
extern void print_memory(Process *p);
extern void print_memory_info(Process *p);
#if defined(DEBUG) && !HEAP_ON_C_STACK
extern Eterm *erts_debug_allocate_tmp_heap(int, Process *);
extern void erts_debug_use_tmp_heap(int, Process *);
extern void erts_debug_unuse_tmp_heap(int, Process *);
#endif
# 100 "beam/erl_debug.h"

#endif /* _ERL_DEBUG_H_ */
# 102 "beam/erl_debug.h"
# 42 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "error.h"
#endif /* expanded by -frewrite-includes */
# 42 "beam/global.h"
# 1 "beam/error.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2010. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERROR_H__
#define __ERROR_H__

/*
 * There are three primary exception classes:
 *
 *	- exit			Process termination - not an error.
 *	- error			Error (adds stacktrace; will be logged).
 *	- thrown		Nonlocal return (turns into a 'nocatch'
 *				error if not caught by the process).
 *
 * In addition, we define a number of exit codes as a convenient
 * short-hand: instead of building the error descriptor term at the time
 * the exception is raised, it is built as necessary when the exception
 * is handled. Examples are EXC_NORMAL, EXC_BADARG, EXC_BADARITH, etc.
 * Some of these have convenient aliases, like BADARG and BADARITH.
 */

/*
 * Bits 0-1 index the 'exception class tag' table.
 */
#define EXC_CLASSBITS 3
#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS)

/*
 * Exception class tags (indices into the 'exception_tag' array)
 */
#define EXTAG_ERROR	0
#define EXTAG_EXIT	1
#define EXTAG_THROWN	2

#define NUMBER_EXC_TAGS 3	/* The number of exception class tags */

/*
 * Exit code flags (bits 2-7)
 *
 * These flags make is easier and quicker to decide what to do with the
 * exception in the early stages, before a handler is found, and also
 * maintains some separation between the class tag and the actions.
 */
#define EXF_PANIC	(1<<2)	/* ignore catches */
#define EXF_THROWN	(1<<3)	/* nonlocal return */
#define EXF_LOG		(1<<4)	/* write to logger on termination */
#define EXF_NATIVE	(1<<5)	/* occurred in native code */
#define EXF_SAVETRACE	(1<<6)	/* save stack trace in internal form */
#define EXF_ARGLIST	(1<<7)	/* has arglist for top of trace */

#define EXC_FLAGBITS 0x00fc

/*
 * The primary fields of an exception code
 */
#define EXF_PRIMARY	(EXF_PANIC | EXF_THROWN | EXF_LOG | EXF_NATIVE)
#define PRIMARY_EXCEPTION(x) ((x) & (EXF_PRIMARY | EXC_CLASSBITS))
#define NATIVE_EXCEPTION(x) ((x) | EXF_NATIVE)

/*
 * Bits 8-12 of the error code are used for indexing into
 * the short-hand error descriptor table.
 */
#define EXC_INDEXBITS 0x1f00
#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> 8)

/*
 * Exit codes used for raising a fresh exception. The primary exceptions
 * share index 0 in the descriptor table. EXC_NULL signals that no
 * exception has occurred. The primary exit codes EXC_EXIT, EXC_ERROR
 * and EXC_THROWN are the basis for all other exit codes, and must
 * always have the EXF_SAVETRACE flag set so that a trace is saved
 * whenever a new exception occurs; the flag is then cleared.
 */
#define EXC_NULL 0			/* Initial value for p->freason */
#define EXC_PRIMARY (0 | EXF_SAVETRACE)
#define EXC_ERROR  (EXC_PRIMARY | EXTAG_ERROR | EXF_LOG)
					/* Generic error (exit term
					 * in p->fvalue) */
#define EXC_EXIT   (EXC_PRIMARY | EXTAG_EXIT)
					/* Generic exit (exit term
					 * in p->fvalue) */
#define EXC_THROWN (EXC_PRIMARY | EXTAG_THROWN | EXF_THROWN)
					/* Generic nonlocal return
					 * (thrown term in p->fvalue) */

#define EXC_ERROR_2 (EXC_ERROR | EXF_ARGLIST)
					/* Error with given arglist term
					 * (exit reason in p->fvalue) */

#define EXC_NORMAL		((1 << 8) | EXC_EXIT)
					/* Normal exit (reason 'normal') */
#define EXC_INTERNAL_ERROR	((2 << 8) | EXC_ERROR | EXF_PANIC)
					/* Things that shouldn't happen */
#define EXC_BADARG		((3 << 8) | EXC_ERROR)
					/* Bad argument to a BIF */
#define EXC_BADARITH		((4 << 8) | EXC_ERROR)
					/* Bad arithmetic */
#define EXC_BADMATCH		((5 << 8) | EXC_ERROR)
					/* Bad match in function body */
#define EXC_FUNCTION_CLAUSE	((6 << 8) | EXC_ERROR)
					 /* No matching function head */
#define EXC_CASE_CLAUSE		((7 << 8) | EXC_ERROR)
					/* No matching case clause */
#define EXC_IF_CLAUSE		((8 << 8) | EXC_ERROR)
					/* No matching if clause */
#define EXC_UNDEF		((9 << 8) | EXC_ERROR)
				 	/* No farity that matches */
#define EXC_BADFUN		((10 << 8) | EXC_ERROR)
					/* Not an existing fun */
#define EXC_BADARITY		((11 << 8) | EXC_ERROR)
					/* Attempt to call fun with
					 * wrong number of arguments. */
#define EXC_TIMEOUT_VALUE	((12 << 8) | EXC_ERROR)
					/* Bad time out value */
#define EXC_NOPROC		((13 << 8) | EXC_ERROR)
					/* No process or port */
#define EXC_NOTALIVE		((14 << 8) | EXC_ERROR)
					/* Not distributed */
#define EXC_SYSTEM_LIMIT	((15 << 8) | EXC_ERROR)
					/* Ran out of something */
#define EXC_TRY_CLAUSE		((16 << 8) | EXC_ERROR)
					/* No matching try clause */
#define EXC_NOTSUP		((17 << 8) | EXC_ERROR)
					/* Not supported */

#define EXC_BADMAP		((18 << 8) | EXC_ERROR)
					/* Bad map */

#define EXC_BADKEY		((19 << 8) | EXC_ERROR)
					/* Bad key in map */

#define NUMBER_EXIT_CODES 20	/* The number of exit code indices */

/*
 * Internal pseudo-error codes.
 */
#define TRAP		(1 << 8)	/* BIF Trap to erlang code */

/*
 * Aliases for some common exit codes.
 */
#define BADARG EXC_BADARG
#define BADARITH EXC_BADARITH
#define BADKEY EXC_BADKEY
#define BADMAP EXC_BADMAP
#define BADMATCH EXC_BADMATCH
#define SYSTEM_LIMIT EXC_SYSTEM_LIMIT


/*
 * Pseudo error codes (these are never seen by the user).
 */
#define TLOAD_OK 0              /* The threaded code linking was successful */
#define TLOAD_MAGIC_NUMBER 1    /* Wrong kind of object file */
#define TLOAD_FORMAT 2          /* Format error while reading object code */
#define TLOAD_MODULE 3          /* Module name in object code does not match */
#define TLOAD_SIZE 4            /* Given size in object code differs from actual size */

/*
 * The exception stack trace parameters.
 */
#define MAX_BACKTRACE_SIZE 64    /* whatever - just not too huge */
#define DEFAULT_BACKTRACE_SIZE 8

/*
 * The table translating an exception code to an atom.
 */
extern Eterm error_atom[NUMBER_EXIT_CODES];

/*
 * The exception tag table.
 */
extern Eterm exception_tag[NUMBER_EXC_TAGS];

/* 
 * The quick-saved stack trace structure
 */
struct StackTrace {
    Eterm header;	/* bignum header - must be first in struct */
    Eterm freason; /* original exception reason is saved in the struct */
    BeamInstr* pc;
    BeamInstr* current;
    int depth;	/* number of saved pointers in trace[] */
    BeamInstr *trace[1];  /* varying size - must be last in struct */
};

#endif /* __ERROR_H__ */
# 206 "beam/error.h"
# 43 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_utils.h"
#endif /* expanded by -frewrite-includes */
# 43 "beam/global.h"
# 44 "beam/global.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 44 "beam/global.h"
# 1 "beam/erl_port.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_PORT_TYPE__
#define ERL_PORT_TYPE__
typedef struct _erl_drv_port Port;
typedef struct ErtsProc2PortSigData_ ErtsProc2PortSigData;
#endif
# 26 "beam/erl_port.h"

#if !defined(ERL_PORT_H__) && !defined(ERL_PORT_GET_PORT_TYPE_ONLY__)
#define ERL_PORT_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_port_task.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port.h"
# 1 "beam/erl_port_task.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2006-2013. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */

/*
 * Description:	Scheduling of port tasks
 *
 * Author: 	Rickard Green
 */

#ifndef ERTS_PORT_TASK_H_BASIC_TYPES__
#define ERTS_PORT_TASK_H_BASIC_TYPES__
#if 0 /* expanded by -frewrite-includes */
#include "erl_sys_driver.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/erl_port_task.h"
# 30 "beam/erl_port_task.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_port_task.h"
# 31 "beam/erl_port_task.h"
#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port_task.h"
# 33 "beam/erl_port_task.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
typedef erts_smp_atomic_t ErtsPortTaskHandle;
#endif
# 36 "beam/erl_port_task.h"

#ifndef ERTS_PORT_TASK_ONLY_BASIC_TYPES__
#ifndef ERL_PORT_TASK_H__
#define ERL_PORT_TASK_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || defined(ERL_IO_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 49 "beam/erl_port_task.h"

#define ERTS_PT_FLG_WAIT_BUSY		(1 << 0)
#define ERTS_PT_FLG_SIG_DEP		(1 << 1)
#define ERTS_PT_FLG_NOSUSPEND		(1 << 2)
#define ERTS_PT_FLG_REF			(1 << 3)
#define ERTS_PT_FLG_BAD_OUTPUT		(1 << 4)

typedef enum {
    ERTS_PORT_TASK_INPUT,
    ERTS_PORT_TASK_OUTPUT,
    ERTS_PORT_TASK_EVENT,
    ERTS_PORT_TASK_TIMEOUT,
    ERTS_PORT_TASK_DIST_CMD,
    ERTS_PORT_TASK_PROC_SIG
} ErtsPortTaskType;

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
/* NOTE: Do not access any of the exported variables directly */
extern erts_smp_atomic_t erts_port_task_outstanding_io_tasks;
#endif
# 69 "beam/erl_port_task.h"

#define ERTS_PTS_FLG_IN_RUNQ			(((erts_aint32_t) 1) <<  0)
#define ERTS_PTS_FLG_EXEC			(((erts_aint32_t) 1) <<  1)
#define ERTS_PTS_FLG_HAVE_TASKS			(((erts_aint32_t) 1) <<  2)
#define ERTS_PTS_FLG_EXIT			(((erts_aint32_t) 1) <<  3)
#define ERTS_PTS_FLG_BUSY_PORT			(((erts_aint32_t) 1) <<  4)
#define ERTS_PTS_FLG_BUSY_PORT_Q		(((erts_aint32_t) 1) <<  5)
#define ERTS_PTS_FLG_CHK_UNSET_BUSY_PORT_Q	(((erts_aint32_t) 1) <<  6)
#define ERTS_PTS_FLG_HAVE_BUSY_TASKS		(((erts_aint32_t) 1) <<  7)
#define ERTS_PTS_FLG_HAVE_NS_TASKS		(((erts_aint32_t) 1) <<  8)
#define ERTS_PTS_FLG_PARALLELISM		(((erts_aint32_t) 1) <<  9)
#define ERTS_PTS_FLG_FORCE_SCHED		(((erts_aint32_t) 1) << 10)
#define ERTS_PTS_FLG_EXITING			(((erts_aint32_t) 1) << 11)
#define ERTS_PTS_FLG_EXEC_IMM			(((erts_aint32_t) 1) << 12)

#define ERTS_PTS_FLGS_BUSY \
    (ERTS_PTS_FLG_BUSY_PORT | ERTS_PTS_FLG_BUSY_PORT_Q)

#define ERTS_PTS_FLGS_FORCE_SCHEDULE_OP		\
    (ERTS_PTS_FLG_EXIT				\
     | ERTS_PTS_FLG_HAVE_BUSY_TASKS		\
     | ERTS_PTS_FLG_HAVE_TASKS			\
     | ERTS_PTS_FLG_EXEC			\
     | ERTS_PTS_FLG_EXEC_IMM			\
     | ERTS_PTS_FLG_FORCE_SCHED			\
     | ERTS_PTS_FLG_EXITING)

#define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH			8192
#define ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW			4096

typedef struct {
    ErlDrvSizeT high;
    erts_smp_atomic_t low;
    erts_smp_atomic_t size;
}  ErtsPortTaskBusyPortQ;

typedef struct ErtsPortTask_ ErtsPortTask;
typedef struct ErtsPortTaskBusyCallerTable_ ErtsPortTaskBusyCallerTable;
typedef struct ErtsPortTaskHandleList_ ErtsPortTaskHandleList;

typedef struct {
    Port *next;
    struct {
	struct {
	    struct {
		ErtsPortTask *first;
		ErtsPortTask *last;
		ErtsPortTaskBusyCallerTable *table;
		ErtsPortTaskHandleList *nosuspend;
	    } busy;
	    ErtsPortTask *first;
	} local;
	struct {
	    ErtsPortTask *first;
	    ErtsPortTask *last;
	} in;
	ErtsPortTaskBusyPortQ *bpq;
    } taskq;
    erts_smp_atomic32_t flags;
#ifdef ERTS_SMP
    erts_mtx_t mtx;
#endif
# 131 "beam/erl_port_task.h"
} ErtsPortTaskSched;

ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp);
ERTS_GLB_INLINE int erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp);
ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
						   ErtsPortTaskBusyPortQ *bpq);
ERTS_GLB_INLINE void erts_port_task_init_sched(ErtsPortTaskSched *ptsp,
					       Eterm id);
ERTS_GLB_INLINE void erts_port_task_fini_sched(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_lock(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE int erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp);
ERTS_GLB_INLINE void erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp);

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void);
#endif
# 148 "beam/erl_port_task.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_port_task_handle_init(ErtsPortTaskHandle *pthp)
{
    erts_smp_atomic_init_nob(pthp, (erts_aint_t) NULL);
}

ERTS_GLB_INLINE int
erts_port_task_is_scheduled(ErtsPortTaskHandle *pthp)
{
    return ((void *) erts_smp_atomic_read_acqb(pthp)) != NULL;
}

ERTS_GLB_INLINE void erts_port_task_pre_init_sched(ErtsPortTaskSched *ptsp,
						   ErtsPortTaskBusyPortQ *bpq)
{
    if (bpq) {
	erts_aint_t low = (erts_aint_t) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_LOW;
	erts_smp_atomic_init_nob(&bpq->low, low);
 	bpq->high = (ErlDrvSizeT) ERTS_PORT_TASK_DEFAULT_BUSY_PORT_Q_HIGH;
	erts_smp_atomic_init_nob(&bpq->size, (erts_aint_t) 0);
    }
    ptsp->taskq.bpq = bpq;
}

ERTS_GLB_INLINE void
erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
{
#ifdef ERTS_SMP
    char *lock_str = "port_sched_lock";
#endif
# 181 "beam/erl_port_task.h"
    ptsp->next = NULL;
    ptsp->taskq.local.busy.first = NULL;
    ptsp->taskq.local.busy.last = NULL;
    ptsp->taskq.local.busy.table = NULL;
    ptsp->taskq.local.busy.nosuspend = NULL;
    ptsp->taskq.local.first = NULL;
    ptsp->taskq.in.first = NULL;
    ptsp->taskq.in.last = NULL;
    erts_smp_atomic32_init_nob(&ptsp->flags, 0);
#ifdef ERTS_SMP
    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id,
#ifdef ERTS_ENABLE_LOCK_COUNT
		    (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
#else
# 195 "beam/erl_port_task.h"
		    1
#endif
# 197 "beam/erl_port_task.h"
		    );
#endif
# 199 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE void
erts_port_task_sched_lock(ErtsPortTaskSched *ptsp)
{
#ifdef ERTS_SMP
    erts_mtx_lock(&ptsp->mtx);
#endif
# 207 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE void
erts_port_task_sched_unlock(ErtsPortTaskSched *ptsp)
{
#ifdef ERTS_SMP
    erts_mtx_unlock(&ptsp->mtx);
#endif
# 215 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE int
erts_port_task_sched_lock_is_locked(ErtsPortTaskSched *ptsp)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    return erts_lc_mtx_is_locked(&ptsp->mtx);
#else
# 223 "beam/erl_port_task.h"
    return 0;
#endif
# 225 "beam/erl_port_task.h"
}


ERTS_GLB_INLINE void
erts_port_task_fini_sched(ErtsPortTaskSched *ptsp)
{
#ifdef ERTS_SMP
    erts_mtx_destroy(&ptsp->mtx);
#endif
# 234 "beam/erl_port_task.h"
}

ERTS_GLB_INLINE void
erts_port_task_sched_enter_exiting_state(ErtsPortTaskSched *ptsp)
{
    erts_smp_atomic32_read_bor_nob(&ptsp->flags, ERTS_PTS_FLG_EXITING);
}

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

ERTS_GLB_INLINE int
erts_port_task_have_outstanding_io_tasks(void)
{
    return (erts_smp_atomic_read_acqb(&erts_port_task_outstanding_io_tasks)
	    != 0);
}

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 252 "beam/erl_port_task.h"

#endif
# 254 "beam/erl_port_task.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
int erts_port_task_execute(ErtsRunQueue *, Port **);
void erts_port_task_init(void);
#endif
# 259 "beam/erl_port_task.h"

void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *);
int erts_port_task_abort(ErtsPortTaskHandle *);

void erts_port_task_abort_nosuspend_tasks(Port *);

int erts_port_task_schedule(Eterm,
			    ErtsPortTaskHandle *,
			    ErtsPortTaskType,
			    ...);
void erts_port_task_free_port(Port *);
int erts_port_is_scheduled(Port *);
ErtsProc2PortSigData *erts_port_task_alloc_p2p_sig_data(void);

#ifdef ERTS_SMP
void erts_enqueue_port(ErtsRunQueue *rq, Port *pp);
Port *erts_dequeue_port(ErtsRunQueue *rq);
#endif
# 277 "beam/erl_port_task.h"
#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif /* ERL_PORT_TASK_H__ */
# 279 "beam/erl_port_task.h"
#endif /* ERTS_PORT_TASK_ONLY_BASIC_TYPES__ */
# 280 "beam/erl_port_task.h"
# 31 "beam/erl_port.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_ptab.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_port.h"
# 1 "beam/erl_ptab.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description:	Process/Port table implementation.
 *
 * Author: 	Rickard Green
 */

#ifndef ERL_PTAB_H__
#define ERL_PTAB_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/erl_ptab.h"
# 31 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/erl_ptab.h"
# 32 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_ptab.h"
# 33 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_utils.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_ptab.h"
# 34 "beam/erl_ptab.h"
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 35 "beam/erl_ptab.h"
# 36 "beam/erl_ptab.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_alloc.h"
#endif /* expanded by -frewrite-includes */
# 37 "beam/erl_ptab.h"
# 38 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 38 "beam/erl_ptab.h"
# 39 "beam/erl_ptab.h"

#define ERTS_TRACER_PROC(P) 	((P)->common.tracer_proc)
#define ERTS_TRACE_FLAGS(P)	((P)->common.trace_flags)

#define ERTS_P_LINKS(P)		((P)->common.u.alive.links)
#define ERTS_P_MONITORS(P)	((P)->common.u.alive.monitors)

#define IS_TRACED(p) \
    (ERTS_TRACER_PROC((p)) != NIL)
#define ARE_TRACE_FLAGS_ON(p,tf) \
    ((ERTS_TRACE_FLAGS((p)) & (tf|F_SENSITIVE)) == (tf))
#define IS_TRACED_FL(p,tf) \
    ( IS_TRACED(p) && ARE_TRACE_FLAGS_ON(p,tf) )

typedef struct {
    Eterm id;
    union {
	erts_atomic_t atmc;
	Sint sint;
    } refc;
    Eterm tracer_proc;
    Uint trace_flags;
    erts_smp_atomic_t timer;
    union {
	/* --- While being alive --- */
	struct {
	    Uint64 started_interval;
	    struct reg_proc *reg;
	    ErtsLink *links;
	    ErtsMonitor *monitors;
	} alive;

	/* --- While being released --- */
	ErtsThrPrgrLaterOp release;
    } u;
} ErtsPTabElementCommon;

typedef struct ErtsPTabDeletedElement_ ErtsPTabDeletedElement;

typedef struct {
    erts_smp_rwmtx_t rwmtx;
    erts_interval_t interval;
    struct {
	ErtsPTabDeletedElement *start;
	ErtsPTabDeletedElement *end;
    } deleted;
    int chunks;
} ErtsPTabListData;

typedef struct {
    erts_smp_atomic64_t last_data;
    erts_smp_atomic32_t count;
    erts_smp_atomic32_t aid_ix;
    erts_smp_atomic32_t fid_ix;
} ErtsPTabVolatileData;

typedef struct {
    erts_smp_atomic_t *tab;
    erts_smp_atomic32_t *free_id_data;
    Uint32 max;
    Uint32 pix_mask;
    Uint32 pix_cl_mask;
    Uint32 pix_cl_shift;
    Uint32 pix_cli_mask;
    Uint32 pix_cli_shift;
    Uint32 dix_cl_mask;
    Uint32 dix_cl_shift;
    Uint32 dix_cli_mask;
    Uint32 dix_cli_shift;
    ErtsPTabElementCommon *invalid_element;
    Eterm invalid_data;
    void (*release_element)(void *);
    UWord element_size;
    int atomic_refc;
} ErtsPTabReadOnlyData;

typedef struct {
    /*
     * Data mainly modified when someone is listing
     * the content of the table.
     */
    union {
	ErtsPTabListData data;
	char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabListData))];
    } list;

    /*
     * Frequently modified data.
     */
    union {
	ErtsPTabVolatileData tile;
	char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabVolatileData))];
    } vola;

    /*
     * Read only data.
     */
    union {
	ErtsPTabReadOnlyData o;
	char algn[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsPTabReadOnlyData))];
    } r;
} ErtsPTab;

#define ERTS_PTAB_ID_DATA_SIZE	28
#define ERTS_PTAB_ID_DATA_SHIFT	(_TAG_IMMED1_SIZE)
/* ERTS_PTAB_MAX_SIZE must be a power of 2 */
#define ERTS_PTAB_MAX_SIZE (SWORD_CONSTANT(1) << 27)
#if (ERTS_PTAB_MAX_SIZE-1) > MAX_SMALL
# error "The maximum number of processes/ports must fit in a SMALL."
#endif
# 149 "beam/erl_ptab.h"


/*
 * Currently pids and ports are allowed.
 */
#if _PID_DATA_SIZE != ERTS_PTAB_ID_DATA_SIZE
# error "Unexpected pid data size"
#endif
# 157 "beam/erl_ptab.h"
#if _PID_DATA_SHIFT != ERTS_PTAB_ID_DATA_SHIFT
# error "Unexpected pid tag size"
#endif
# 160 "beam/erl_ptab.h"
#if _PORT_DATA_SIZE != ERTS_PTAB_ID_DATA_SIZE
# error "Unexpected port data size"
#endif
# 163 "beam/erl_ptab.h"
#if _PORT_DATA_SHIFT != ERTS_PTAB_ID_DATA_SHIFT
# error "Unexpected port tag size"
#endif
# 166 "beam/erl_ptab.h"

#define ERTS_PTAB_INVALID_ID(TAG)					\
    ((Eterm)								\
     ((((1 << ERTS_PTAB_ID_DATA_SIZE) - 1) << ERTS_PTAB_ID_DATA_SHIFT)	\
      | (TAG)))

#define erts_ptab_is_valid_id(ID)					\
    (is_internal_pid((ID)) || is_internal_port((ID)))

void erts_ptab_init(void);
void erts_ptab_init_table(ErtsPTab *ptab,
			  ErtsAlcType_t atype,
			  void (*release_element)(void *),
			  ErtsPTabElementCommon *invalid_element,
			  int size,
			  UWord element_size,
			  char *name,
			  int legacy,
			  int atomic_refc);
int erts_ptab_new_element(ErtsPTab *ptab,
			  ErtsPTabElementCommon *ptab_el,
			  void *init_arg,
			  void (*init_ptab_el)(void *, Eterm));
void erts_ptab_delete_element(ErtsPTab *ptab,
			      ErtsPTabElementCommon *ptab_el);
int erts_ptab_initialized(ErtsPTab *ptab);
UWord erts_ptab_mem_size(ErtsPTab *ptab);

ERTS_GLB_INLINE erts_interval_t *erts_ptab_interval(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_count(ErtsPTab *ptab);
ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata);
ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata);
ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data);
ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data);
ERTS_GLB_INLINE Eterm erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag);
ERTS_GLB_INLINE int erts_ptab_id2pix(ErtsPTab *ptab, Eterm id);
ERTS_GLB_INLINE Uint erts_ptab_id2data(ErtsPTab *ptab, Eterm id);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix);
ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
					     Sint add_refc);
ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el,
						  Sint add_refc);
ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el);
ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab);
ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab);
ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab);
ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab);
ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_interval_t *
erts_ptab_interval(ErtsPTab *ptab)
{
    return &ptab->list.data.interval;
}

ERTS_GLB_INLINE int
erts_ptab_max(ErtsPTab *ptab)
{
    int max = ptab->r.o.max;
    return max == ERTS_PTAB_MAX_SIZE ? max - 1 : max;
}

ERTS_GLB_INLINE int
erts_ptab_count(ErtsPTab *ptab)
{
    int max = ptab->r.o.max;
    erts_aint32_t res = erts_smp_atomic32_read_nob(&ptab->vola.tile.count);
    if (max == ERTS_PTAB_MAX_SIZE) {
	max--;
	res--;
    }
    if (res > max)
	return max;
    ASSERT(res >= 0);
    return (int) res;

}

ERTS_GLB_INLINE Uint erts_ptab_pixdata2data(ErtsPTab *ptab, Eterm pixdata)
{
    Uint32 data = ((Uint32) pixdata) & ~ptab->r.o.pix_mask;
    data |= (pixdata >> ptab->r.o.pix_cl_shift) & ptab->r.o.pix_cl_mask;
    data |= (pixdata & ptab->r.o.pix_cli_mask) << ptab->r.o.pix_cli_shift;
    return data;
}

ERTS_GLB_INLINE Uint32 erts_ptab_pixdata2pix(ErtsPTab *ptab, Eterm pixdata)
{
    return ((Uint32) pixdata) & ptab->r.o.pix_mask;
}

ERTS_GLB_INLINE Uint32 erts_ptab_data2pix(ErtsPTab *ptab, Eterm data)
{
    Uint32 n, pix;
    n = (Uint32) data;
    pix = ((n & ptab->r.o.pix_cl_mask) << ptab->r.o.pix_cl_shift);
    pix += ((n >> ptab->r.o.pix_cli_shift) & ptab->r.o.pix_cli_mask);
    ASSERT(0 <= pix && pix < ptab->r.o.max);
    return pix;
}

ERTS_GLB_INLINE Uint erts_ptab_data2pixdata(ErtsPTab *ptab, Eterm data)
{
    Uint pixdata = data & ~((Uint) ptab->r.o.pix_mask);
    pixdata |= (Uint) erts_ptab_data2pix(ptab, data);
    ASSERT(data == erts_ptab_pixdata2data(ptab, pixdata));
    return pixdata;
}

#if ERTS_SIZEOF_TERM == 8

ERTS_GLB_INLINE Eterm
erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag)
{
    HUint huint;
    Uint32 low_data = (Uint32) data;
    low_data &= (1 << ERTS_PTAB_ID_DATA_SIZE) - 1;
    low_data <<= ERTS_PTAB_ID_DATA_SHIFT;
    huint.hval[ERTS_HUINT_HVAL_HIGH] = erts_ptab_data2pix(ptab, data);
    huint.hval[ERTS_HUINT_HVAL_LOW] = low_data | ((Uint32) tag);
    return (Eterm) huint.val;
}

ERTS_GLB_INLINE int
erts_ptab_id2pix(ErtsPTab *ptab, Eterm id)
{
    HUint huint;
    huint.val = id;
    return (int) huint.hval[ERTS_HUINT_HVAL_HIGH];
}

ERTS_GLB_INLINE Uint
erts_ptab_id2data(ErtsPTab *ptab, Eterm id)
{
    HUint huint;
    huint.val = id;
    return (Uint) (huint.hval[ERTS_HUINT_HVAL_LOW] >> ERTS_PTAB_ID_DATA_SHIFT);
}

#elif ERTS_SIZEOF_TERM == 4
# 320 "beam/erl_ptab.h"

ERTS_GLB_INLINE Eterm
erts_ptab_make_id(ErtsPTab *ptab, Eterm data, Eterm tag)
{
    Eterm id;
    data &= ((1 << ERTS_PTAB_ID_DATA_SIZE) - 1);
    id = (Eterm) erts_ptab_data2pixdata(ptab, data);
    return (id << ERTS_PTAB_ID_DATA_SHIFT) | tag;
}

ERTS_GLB_INLINE int
erts_ptab_id2pix(ErtsPTab *ptab, Eterm id)
{
    Uint pixdata = (Uint) id;
    pixdata >>= ERTS_PTAB_ID_DATA_SHIFT;
    return (int) erts_ptab_pixdata2pix(ptab, pixdata);
}

ERTS_GLB_INLINE Uint
erts_ptab_id2data(ErtsPTab *ptab, Eterm id)
{
    Uint pixdata = (Uint) id;
    pixdata >>= ERTS_PTAB_ID_DATA_SHIFT;
    return erts_ptab_pixdata2data(ptab, pixdata);
}

#else
# 347 "beam/erl_ptab.h"
#error "Unsupported size of term"
#endif
# 349 "beam/erl_ptab.h"

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_nob(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_ddrb(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_ddrb(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_rb(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_rb(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE erts_aint_t erts_ptab_pix2intptr_acqb(ErtsPTab *ptab, int ix)
{
    ASSERT(0 <= ix && ix < ptab->r.o.max);
    return erts_smp_atomic_read_acqb(&ptab->r.o.tab[ix]);
}

ERTS_GLB_INLINE void erts_ptab_atmc_inc_refc(ErtsPTabElementCommon *ptab_el)
{
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_aint_t refc = erts_atomic_inc_read_nob(&ptab_el->refc.atmc);
    ERTS_LC_ASSERT(refc > 1);
#else
# 380 "beam/erl_ptab.h"
    erts_atomic_inc_nob(&ptab_el->refc.atmc);
#endif
# 382 "beam/erl_ptab.h"
}

ERTS_GLB_INLINE Sint erts_ptab_atmc_dec_test_refc(ErtsPTabElementCommon *ptab_el)
{
    erts_aint_t refc = erts_atomic_dec_read_relb(&ptab_el->refc.atmc);
    ERTS_SMP_LC_ASSERT(refc >= 0);
#ifdef ERTS_SMP
    if (refc == 0)
	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif
# 392 "beam/erl_ptab.h"
    return (Sint) refc;
}

ERTS_GLB_INLINE Sint erts_ptab_atmc_add_test_refc(ErtsPTabElementCommon *ptab_el,
						  Sint add_refc)
{
    erts_aint_t refc = erts_atomic_add_read_mb(&ptab_el->refc.atmc,
					       (erts_aint_t) add_refc);
    ERTS_SMP_LC_ASSERT(refc >= 0);
    return (Sint) refc;
}

ERTS_GLB_INLINE Sint erts_ptab_atmc_read_refc(ErtsPTabElementCommon *ptab_el)
{
    return (Sint) erts_atomic_read_nob(&ptab_el->refc.atmc);
}

ERTS_GLB_INLINE void erts_ptab_inc_refc(ErtsPTabElementCommon *ptab_el)
{
    ptab_el->refc.sint++;
    ASSERT(ptab_el->refc.sint > 1);
}

ERTS_GLB_INLINE Sint erts_ptab_dec_test_refc(ErtsPTabElementCommon *ptab_el)
{
    Sint refc = --ptab_el->refc.sint;
    ERTS_SMP_LC_ASSERT(refc >= 0);
    return refc;
}

ERTS_GLB_INLINE Sint erts_ptab_add_test_refc(ErtsPTabElementCommon *ptab_el,
					     Sint add_refc)
{
    ptab_el->refc.sint += add_refc;
    ERTS_SMP_LC_ASSERT(ptab_el->refc.sint >= 0);
    return (Sint) ptab_el->refc.sint;
}

ERTS_GLB_INLINE Sint erts_ptab_read_refc(ErtsPTabElementCommon *ptab_el)
{
    return ptab_el->refc.sint;
}

ERTS_GLB_INLINE void erts_ptab_rlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_rlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_ptab_tryrlock(ErtsPTab *ptab)
{
    return erts_smp_rwmtx_tryrlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE void erts_ptab_runlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_runlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE void erts_ptab_rwlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_rwlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_ptab_tryrwlock(ErtsPTab *ptab)
{
    return erts_smp_rwmtx_tryrwlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE void erts_ptab_rwunlock(ErtsPTab *ptab)
{
    erts_smp_rwmtx_rwunlock(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rlocked(ErtsPTab *ptab)
{
    return erts_smp_lc_rwmtx_is_rlocked(&ptab->list.data.rwmtx);
}

ERTS_GLB_INLINE int erts_smp_lc_ptab_is_rwlocked(ErtsPTab *ptab)
{
    return erts_smp_lc_rwmtx_is_rwlocked(&ptab->list.data.rwmtx);
}

#endif
# 476 "beam/erl_ptab.h"

#endif
# 478 "beam/erl_ptab.h"

#if defined(ERTS_PTAB_WANT_BIF_IMPL__) && !defined(ERTS_PTAB_LIST__)
#define ERTS_PTAB_LIST__

#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 482 "beam/erl_ptab.h"
# 483 "beam/erl_ptab.h"
#if 0 /* expanded by -frewrite-includes */
#include "bif.h"
#endif /* expanded by -frewrite-includes */
# 483 "beam/erl_ptab.h"
# 484 "beam/erl_ptab.h"

BIF_RETTYPE erts_ptab_list(struct process *c_p, ErtsPTab *ptab);

#endif
# 488 "beam/erl_ptab.h"

#if defined(ERTS_PTAB_WANT_DEBUG_FUNCS__) && !defined(ERTS_PTAB_DEBUG_FUNCS__)
#define ERTS_PTAB_DEBUG_FUNCS__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 491 "beam/erl_ptab.h"
# 492 "beam/erl_ptab.h"

/* Debug functions */
Sint erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next);
Eterm erts_debug_ptab_list(Process *c_p, ErtsPTab *ptab);
Eterm erts_debug_ptab_list_bif_info(Process *c_p, ErtsPTab *ptab);

#endif
# 499 "beam/erl_ptab.h"
# 32 "beam/erl_port.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/erl_port.h"
# 1 "beam/erl_thr_progress.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread progress information. Used by lock free algorithms
 *              to determine when all involved threads are guaranteed to
 *              have passed a specific point of execution.
 *
 *              Usage instructions can be found in ert_thr_progress.c
 *
 * Author: 	Rickard Green
 */

#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
#define ERL_THR_PROGRESS_H__TSD_TYPE__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/erl_thr_progress.h"
# 35 "beam/erl_thr_progress.h"

#ifndef ERTS_SMP

#define erts_smp_thr_progress_block() ((void) 0)
#define erts_smp_thr_progress_unblock() ((void) 0)
#define erts_smp_thr_progress_is_blocking() 1

#else /* ERTS_SMP */
# 43 "beam/erl_thr_progress.h"

#define erts_smp_thr_progress_block erts_thr_progress_block
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking

void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);

typedef Uint64 ErtsThrPrgrVal;

#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */

typedef struct {
    ErtsThrPrgrVal next;
    ErtsThrPrgrVal current;
    int chk_next_ix;
    struct {
	int current;
	int waiting;
    } umrefc_ix;
} ErtsThrPrgrLeaderState;

typedef struct {
    int id;
    int is_managed;
    int is_blocking;
#ifdef ERTS_ENABLE_LOCK_CHECK
    int is_delaying; /* managed is always delaying */
#endif
# 73 "beam/erl_thr_progress.h"
    int is_temporary;

    /* --- Part below only for registered threads --- */

    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];

    /* --- Part below only for managed threads --- */

    int leader; /* Needs to be first in the managed threads part */
    int active;
    ErtsThrPrgrVal confirmed;
    ErtsThrPrgrLeaderState leader_state;
} ErtsThrPrgrData;

int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
void erts_thr_progress_fatal_error_wait(SWord timeout);

#endif /* ERTS_SMP */
# 91 "beam/erl_thr_progress.h"

typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
struct ErtsThrPrgrLaterOp_ {
#ifdef ERTS_SMP
    ErtsThrPrgrVal later;
#endif
# 97 "beam/erl_thr_progress.h"
    void (*func)(void *);
    void *data;
    ErtsThrPrgrLaterOp *next;
};

#endif
# 103 "beam/erl_thr_progress.h"

#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
#define ERL_THR_PROGRESS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 107 "beam/erl_thr_progress.h"
# 1 "beam/erl_threads.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Description: Error checking thread interface to the ethread library.
 *              All functions terminates the emulator on failure.
 * Author: Rickard Green
 */

#ifndef ERL_THREAD_H__
#define ERL_THREAD_H__

/*
 * --- Documentation of atomics and memory barriers --------------------------
 *
 * The following explicit memory barriers exist:
 *   
 * - ERTS_THR_MEMORY_BARRIER
 *      Full memory barrier. Orders both loads, and stores. No
 *      load or store is allowed to be reordered over the
 *      barrier.
 * - ERTS_THR_WRITE_MEMORY_BARRIER
 *      Write barrier. Orders *only* stores. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_READ_MEMORY_BARRIER
 *      Read barrier. Orders *only* loads. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 *      Data dependency read barrier. Orders *only* loads
 *      according to data dependency across the barrier.
 *
 * If thread support has been disabled, these barriers will become no-ops.
 *
 * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
 * be enabled only in the SMP enabled runtime system.
 *
 * --- Atomic operations ---
 *
 * Atomics operations exist for 32-bit, word size, and double word size
 * integers. Function prototypes are listed below.
 *
 * Each function implementing an atomic operation exist with the following
 * implied memory barrier semantics. Not all combinations are useful, but
 * all of them exist for simplicity. <B> is suffix in function name:
 *
 * - <B>  - Description
 *
 * - mb   - Full memory barrier. Orders both loads, and
 *          stores before, and after the atomic operation.
 *          No load or store is allowed to be reordered
 *          over the atomic operation.
 * - relb - Release barrier. Orders both loads, and
 *          stores appearing *before* the atomic
 *          operation. These are not allowed to be
 *          reordered over the atomic operation.
 * - acqb - Acquire barrier. Orders both loads, and stores
 *          appearing *after* the atomic operation. These
 *          are not allowed to be reordered over the
 *          atomic operation.
 * - wb   - Write barrier. Orders *only* stores. These are
 *          not allowed to be reordered over the barrier.
 *          Store in atomic operation is ordered *after*
 *          the barrier.
 * - rb   - Read barrier. Orders *only* loads. These are
 *          not allowed to be reordered over the barrier.
 *          Load in atomic operation is ordered *before*
 *          the barrier. 
 * - ddrb - Data dependency read barrier. Orders *only*
 *          loads according to data dependency across the
 *          barrier. Load in atomic operation is ordered
 *          before the barrier.
 *
 * If thread support has been disabled, these functions are mapped to
 * functions that performs the same operation, but aren't atomic
 * and don't imply any memory barriers.
 *
 * If the atomic operations are prefixed with erts_smp_ instead of only
 * erts_ the atomic operations will only be atomic in the SMP enabled
 * runtime system, and will be mapped to non-atomic operations without
 * memory barriers in the runtime system without SMP support. Atomic
 * operations with erts_smp_ prefix should use the atomic types
 * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
 * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
 * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
 * are the same.
 *
 * --- 32-bit atomic operations ---
 *
 * The following 32-bit atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above). Note
 * that sizeof(erts_atomic32_t) might be larger than 4!
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
 *                               erts_aint32_t val);
 *
 * Set value:
 *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Read; returns current value:
 *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment:
 *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
 *
 * Decrement:
 *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Add value:
 *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
 *                                             erts_aint32_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
 *                                        erts_aint32_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
 *                                           erts_aint32_t new,
 *                                           erts_aint32_t exp);
 *
 * --- Word size atomic operations ---
 *
 * The following word size (same size as sizeof(void *)) atomic
 * operations exist. <B> should be replaced with a supported
 * memory barrier (see above). Note that sizeof(erts_atomic_t)
 * might be larger than sizeof(void *)!
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
 *                             erts_aint_t val);
 *
 * Set value;
 *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Read; returns current value:
 *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
 *
 * Increment:
 *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
 *
 * Decrement:
 *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Add value:
 *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
 *                                         erts_aint_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
 *                                    erts_aint_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
 *                                       erts_aint_t new,
 *                                       erts_aint_t exp);
 *
 * --- Double word size atomic operations ---
 *
 * The following double word atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above).
 *
 * Note that sizeof(erts_dw_atomic_t) usually is larger than
 * 2*sizeof(void *)!
 * 
 * The erts_dw_aint_t data type should be accessed as if it was defined
 * like this:
 *
 *     typedef struct {
 *         erts_aint_t sint[2];
 *     } erts_dw_aint_t;
 *
 *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
 *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
 *                           erts_dw_aint_t *val);
 *
 * Set; value is written into 'val':
 *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
 *                               erts_dw_aint_t *val);
 *
 * Read; value is written into 'val':
 *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
 *                                erts_dw_aint_t *val);
 *
 * Compare and exchange; returns a value != 0 if exchange was
 * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
 * contains the same value as in memory when the function is called,
 * 'new' is written to memory; otherwise, not. If exchange was not
 * made, 'exp_act' contains the actual value in memory:
 *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
 *                                  erts_dw_aint_t *new,
 *                                  erts_dw_aint_t *exp_act);
 */

#define ERTS_SPIN_BODY ETHR_SPIN_BODY

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 260 "beam/erl_threads.h"
# 261 "beam/erl_threads.h"

#ifdef USE_THREADS

#define ETHR_TRY_INLINE_FUNCS
#if 0 /* expanded by -frewrite-includes */
#include "ethread.h"
#endif /* expanded by -frewrite-includes */
# 265 "beam/erl_threads.h"
# 266 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 266 "beam/erl_threads.h"
# 267 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_count.h"
#endif /* expanded by -frewrite-includes */
# 267 "beam/erl_threads.h"
# 268 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 268 "beam/erl_threads.h"
# 269 "beam/erl_threads.h"

#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
/*
 * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
 * only seen this bug in glibc versions before 2.4. Note that condition
 * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
 * this bug since these implementations may use mutexes internally.
 */
#  define ERTS_THR_HAVE_BUSY_DESTROY_BUG
#endif
# 279 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__)
#define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__)
#endif
# 296 "beam/erl_threads.h"

#define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
typedef ethr_thr_opts erts_thr_opts_t;
typedef ethr_init_data erts_thr_init_data_t;
typedef ethr_late_init_data erts_thr_late_init_data_t;
typedef ethr_tid erts_tid_t;

/* mutex */
typedef struct {
    ethr_mutex mtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 309 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 312 "beam/erl_threads.h"

} erts_mtx_t;
typedef ethr_cond erts_cnd_t;

/* rwmutex */
typedef struct {
    ethr_rwmutex rwmtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 322 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 325 "beam/erl_threads.h"
} erts_rwmtx_t;

#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER
#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
typedef ethr_rwmutex_opt erts_rwmtx_opt_t;

typedef ethr_tsd_key erts_tsd_key_t;
typedef ethr_ts_event erts_tse_t;
#define erts_dw_aint_t ethr_dw_sint_t
#define erts_dw_atomic_t ethr_dw_atomic_t
#define erts_aint_t ethr_sint_t
#define erts_atomic_t ethr_atomic_t
#define erts_aint32_t ethr_sint32_t 
#define erts_atomic32_t ethr_atomic32_t

#if defined(ARCH_32)
#  define erts_atomic64_t ethr_dw_atomic_t
#  define erts_aint64_t ethr_sint64_t
#elif defined(ARCH_64)
# 352 "beam/erl_threads.h"
#  define erts_atomic64_t ethr_atomic_t
#  define erts_aint64_t ethr_sint_t
#else
# 355 "beam/erl_threads.h"
#  error "Not supported architecture"
#endif
# 357 "beam/erl_threads.h"

#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD

/* spinlock */
typedef struct {
    ethr_spinlock_t slck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 367 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 370 "beam/erl_threads.h"
} erts_spinlock_t;

/* rwlock */
typedef struct {
    ethr_rwlock_t rwlck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 378 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 381 "beam/erl_threads.h"
} erts_rwlock_t;

__decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
                                 /* implemented in erl_init.c */

#define ERTS_THR_INIT_DATA_DEF_INITER	ETHR_INIT_DATA_DEFAULT_INITER
#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
                                        ETHR_LATE_INIT_DATA_DEFAULT_INITER

#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
#  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
#endif
# 393 "beam/erl_threads.h"

#else /* #ifdef USE_THREADS */
# 395 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#define ERTS_THR_OPTS_DEFAULT_INITER 0
typedef int erts_thr_opts_t;
typedef int erts_thr_init_data_t;
typedef int erts_thr_late_init_data_t;
typedef int erts_tid_t;
typedef int erts_mtx_t;
typedef int erts_cnd_t;
#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_RWMTX_TYPE_NORMAL 0
#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_RWMTX_LONG_LIVED 0
#define ERTS_RWMTX_SHORT_LIVED 0
#define ERTS_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_rwmtx_opt_t;
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;

typedef struct { SWord sint[2]; } erts_dw_aint_t;
typedef SWord erts_aint_t;
typedef Sint32 erts_aint32_t;
typedef Sint64 erts_aint64_t;

#define erts_dw_atomic_t erts_dw_aint_t
#define erts_atomic_t erts_aint_t
#define erts_atomic32_t erts_aint32_t
#define erts_atomic64_t erts_aint64_t

#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
#else
# 439 "beam/erl_threads.h"
typedef struct { int gcc_is_buggy; } erts_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif
# 442 "beam/erl_threads.h"

#ifdef WORDS_BIGENDIAN
#define ERTS_DW_AINT_LOW_WORD 1
#define ERTS_DW_AINT_HIGH_WORD 0
#else
# 447 "beam/erl_threads.h"
#define ERTS_DW_AINT_LOW_WORD 0
#define ERTS_DW_AINT_HIGH_WORD 1
#endif
# 450 "beam/erl_threads.h"

#define ERTS_MTX_INITER			0
#define ERTS_CND_INITER			0
#define ERTS_THR_INIT_DATA_DEF_INITER	0

#define ERTS_HAVE_REC_MTX_INIT		1

#endif /* #ifdef USE_THREADS */
# 458 "beam/erl_threads.h"

#define erts_no_dw_atomic_t erts_dw_aint_t
#define erts_no_atomic_t erts_aint_t
#define erts_no_atomic32_t erts_aint32_t
#define erts_no_atomic64_t erts_aint64_t

#define ERTS_AINT_NULL ((erts_aint_t) NULL)

#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))

ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
				     void *arg, erts_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid);
ERTS_GLB_INLINE void erts_thr_exit(void *res);
ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
				     int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
					 Uint16 opt, int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
					    char *name,
					    Eterm extra,
					    int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file,
				       unsigned int line);
ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file,
				     unsigned int line);
#else
# 499 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx);
#endif
# 502 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
					   erts_rwmtx_opt_t *opt,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
				       char *name,
				       Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
					 erts_rwmtx_opt_t *opt,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 529 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
#endif
# 534 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);

ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *val,
					      erts_no_dw_atomic_t *old_val);
ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
						    erts_aint_t i);
ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
						    erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
						     erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
						erts_aint_t new);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
						   erts_aint_t new,
						   erts_aint_t expected);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var,
						     erts_aint_t mask,
						     erts_aint_t set);
ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
							erts_aint32_t i);
ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
							erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
							 erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
						    erts_aint32_t new);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
						       erts_aint32_t new,
						       erts_aint32_t expected);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
							 erts_aint32_t mask,
							 erts_aint32_t set);
ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
							erts_aint64_t i);
ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
							erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
							 erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
						    erts_aint64_t new);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
						       erts_aint64_t new,
						       erts_aint64_t expected);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
							 erts_aint64_t mask,
							 erts_aint64_t set);

ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
					      char *name,
					      Eterm extra,
					      Uint16 opt);
ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
					  char *name,
					  Eterm extra);
ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
					char *name);
ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line);
#else
# 626 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
#endif
# 628 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_rwlock_init_x(erts_rwlock_t *lock,
					char *name,
					Eterm extra);
ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
				      char *name);
ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
#else
# 640 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_read_lock(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
#endif
# 643 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname);
ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
ERTS_GLB_INLINE int erts_thr_get_main_status(void);
ERTS_GLB_INLINE void erts_thr_yield(void);


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS
#define ERTS_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
				      sigset_t *oset);
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);

#ifdef USE_THREADS
ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
#endif
# 674 "beam/erl_threads.h"

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 676 "beam/erl_threads.h"

#ifdef USE_THREADS

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \
do {									\
    Type act = ReadOp((VarP));						\
    while (1) {								\
	Type exp = act;							\
	Type new = exp & ~(Mask);					\
	new |= ((Mask) & (Set));					\
	act = CmpxchgOp((VarP), new, exp);				\
	if (act == exp)							\
	    return act;							\
    }									\
} while (0)
#endif
# 750 "beam/erl_threads.h"

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var);
ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var);

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of this file for info on atomics.
 */

/* Double word size atomics */

#define erts_dw_atomic_init_nob ethr_dw_atomic_init
#define erts_dw_atomic_set_nob ethr_dw_atomic_set
#define erts_dw_atomic_read_nob ethr_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb

#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb

#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb

#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb

#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb

#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    sint[0] = val->sint[0];
    sint[1] = val->sint[1];    
}

ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    val->sint[0] = sint[0];
    val->sint[1] = sint[1];
}

#endif
# 825 "beam/erl_threads.h"

/* Word size atomics */

#define erts_atomic_init_nob ethr_atomic_init
#define erts_atomic_set_nob ethr_atomic_set
#define erts_atomic_read_nob ethr_atomic_read
#define erts_atomic_inc_read_nob ethr_atomic_inc_read
#define erts_atomic_dec_read_nob ethr_atomic_dec_read
#define erts_atomic_inc_nob ethr_atomic_inc
#define erts_atomic_dec_nob ethr_atomic_dec
#define erts_atomic_add_read_nob ethr_atomic_add_read
#define erts_atomic_add_nob ethr_atomic_add
#define erts_atomic_read_bor_nob ethr_atomic_read_bor
#define erts_atomic_read_band_nob ethr_atomic_read_band
#define erts_atomic_xchg_nob ethr_atomic_xchg
#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg,
			    var, mask, set);
}
#endif
# 854 "beam/erl_threads.h"

#define erts_atomic_init_mb ethr_atomic_init_mb
#define erts_atomic_set_mb ethr_atomic_set_mb
#define erts_atomic_read_mb ethr_atomic_read_mb
#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic_inc_mb ethr_atomic_inc_mb
#define erts_atomic_dec_mb ethr_atomic_dec_mb
#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic_add_mb ethr_atomic_add_mb
#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 881 "beam/erl_threads.h"

#define erts_atomic_init_acqb ethr_atomic_init_acqb
#define erts_atomic_set_acqb ethr_atomic_set_acqb
#define erts_atomic_read_acqb ethr_atomic_read_acqb
#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic_add_acqb ethr_atomic_add_acqb
#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 908 "beam/erl_threads.h"

#define erts_atomic_init_relb ethr_atomic_init_relb
#define erts_atomic_set_relb ethr_atomic_set_relb
#define erts_atomic_read_relb ethr_atomic_read_relb
#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic_inc_relb ethr_atomic_inc_relb
#define erts_atomic_dec_relb ethr_atomic_dec_relb
#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic_add_relb ethr_atomic_add_relb
#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 935 "beam/erl_threads.h"

#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 962 "beam/erl_threads.h"

#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic_inc_rb ethr_atomic_inc_rb
#define erts_atomic_dec_rb ethr_atomic_dec_rb
#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic_add_rb ethr_atomic_add_rb
#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 989 "beam/erl_threads.h"

#define erts_atomic_init_wb ethr_atomic_init_wb
#define erts_atomic_set_wb ethr_atomic_set_wb
#define erts_atomic_read_wb ethr_atomic_read_wb
#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic_inc_wb ethr_atomic_inc_wb
#define erts_atomic_dec_wb ethr_atomic_dec_wb
#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic_add_wb ethr_atomic_add_wb
#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1018 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    return *sint;
}

#endif
# 1036 "beam/erl_threads.h"

/* 32-bit atomics */

#define erts_atomic32_init_nob ethr_atomic32_init
#define erts_atomic32_set_nob ethr_atomic32_set
#define erts_atomic32_read_nob ethr_atomic32_read
#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
#define erts_atomic32_inc_nob ethr_atomic32_inc
#define erts_atomic32_dec_nob ethr_atomic32_dec
#define erts_atomic32_add_read_nob ethr_atomic32_add_read
#define erts_atomic32_add_nob ethr_atomic32_add
#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
#define erts_atomic32_read_band_nob ethr_atomic32_read_band
#define erts_atomic32_xchg_nob ethr_atomic32_xchg
#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg,
			    var, mask, set);
}
#endif
# 1065 "beam/erl_threads.h"

#define erts_atomic32_init_mb ethr_atomic32_init_mb
#define erts_atomic32_set_mb ethr_atomic32_set_mb
#define erts_atomic32_read_mb ethr_atomic32_read_mb
#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
#define erts_atomic32_add_mb ethr_atomic32_add_mb
#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 1092 "beam/erl_threads.h"

#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 1119 "beam/erl_threads.h"

#define erts_atomic32_init_relb ethr_atomic32_init_relb
#define erts_atomic32_set_relb ethr_atomic32_set_relb
#define erts_atomic32_read_relb ethr_atomic32_read_relb
#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
#define erts_atomic32_add_relb ethr_atomic32_add_relb
#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 1146 "beam/erl_threads.h"

#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 1173 "beam/erl_threads.h"

#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
#define erts_atomic32_add_rb ethr_atomic32_add_rb
#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 1200 "beam/erl_threads.h"

#define erts_atomic32_init_wb ethr_atomic32_init_wb
#define erts_atomic32_set_wb ethr_atomic32_set_wb
#define erts_atomic32_read_wb ethr_atomic32_read_wb
#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
#define erts_atomic32_add_wb ethr_atomic32_add_wb
#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1229 "beam/erl_threads.h"

#undef ERTS_ATOMIC_BSET_IMPL__

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    return *sint;
}

#endif
# 1249 "beam/erl_threads.h"

/* 64-bit atomics */

#if defined(ARCH_64)

#define erts_atomic64_init_nob ethr_atomic_init
#define erts_atomic64_set_nob ethr_atomic_set
#define erts_atomic64_read_nob ethr_atomic_read
#define erts_atomic64_inc_read_nob ethr_atomic_inc_read
#define erts_atomic64_dec_read_nob ethr_atomic_dec_read
#define erts_atomic64_inc_nob ethr_atomic_inc
#define erts_atomic64_dec_nob ethr_atomic_dec
#define erts_atomic64_add_read_nob ethr_atomic_add_read
#define erts_atomic64_add_nob ethr_atomic_add
#define erts_atomic64_read_bor_nob ethr_atomic_read_bor
#define erts_atomic64_read_band_nob ethr_atomic_read_band
#define erts_atomic64_xchg_nob ethr_atomic_xchg
#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob

#define erts_atomic64_init_mb ethr_atomic_init_mb
#define erts_atomic64_set_mb ethr_atomic_set_mb
#define erts_atomic64_read_mb ethr_atomic_read_mb
#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic64_inc_mb ethr_atomic_inc_mb
#define erts_atomic64_dec_mb ethr_atomic_dec_mb
#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic64_add_mb ethr_atomic_add_mb
#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb

#define erts_atomic64_init_acqb ethr_atomic_init_acqb
#define erts_atomic64_set_acqb ethr_atomic_set_acqb
#define erts_atomic64_read_acqb ethr_atomic_read_acqb
#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic64_add_acqb ethr_atomic_add_acqb
#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_atomic64_init_relb ethr_atomic_init_relb
#define erts_atomic64_set_relb ethr_atomic_set_relb
#define erts_atomic64_read_relb ethr_atomic_read_relb
#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic64_inc_relb ethr_atomic_inc_relb
#define erts_atomic64_dec_relb ethr_atomic_dec_relb
#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic64_add_relb ethr_atomic_add_relb
#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb

#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_atomic64_init_rb ethr_atomic_init_rb
#define erts_atomic64_set_rb ethr_atomic_set_rb
#define erts_atomic64_read_rb ethr_atomic_read_rb
#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic64_inc_rb ethr_atomic_inc_rb
#define erts_atomic64_dec_rb ethr_atomic_dec_rb
#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic64_add_rb ethr_atomic_add_rb
#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb

#define erts_atomic64_init_wb ethr_atomic_init_wb
#define erts_atomic64_set_wb ethr_atomic_set_wb
#define erts_atomic64_read_wb ethr_atomic_read_wb
#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic64_inc_wb ethr_atomic_inc_wb
#define erts_atomic64_dec_wb ethr_atomic_dec_wb
#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic64_add_wb ethr_atomic_add_wb
#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb

#define erts_atomic64_set_dirty erts_atomic_set_dirty
#define erts_atomic64_read_dirty erts_atomic_read_dirty

#elif defined(ARCH_32)
# 1363 "beam/erl_threads.h"

#undef ERTS_ATOMIC64_OPS_DECL__

#define ERTS_ATOMIC64_OPS_DECL__(BARRIER)				\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)

ERTS_ATOMIC64_OPS_DECL__(nob);
ERTS_ATOMIC64_OPS_DECL__(mb);
ERTS_ATOMIC64_OPS_DECL__(acqb);
ERTS_ATOMIC64_OPS_DECL__(relb);
ERTS_ATOMIC64_OPS_DECL__(ddrb);
ERTS_ATOMIC64_OPS_DECL__(rb);
ERTS_ATOMIC64_OPS_DECL__(wb);

#undef ERTS_ATOMIC64_OPS_DECL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

/*
 * The ethr_dw_atomic_*_nob() functions below
 * are here to make it possible for the
 * ERTS_ATOMIC64_OPS_IMPL__() to map erts
 * barriers to ethread barriers...
 */
static ERTS_INLINE void
ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_init(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
		       ethr_dw_sint_t *val)
{
    ethr_dw_atomic_set(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_read(var, val);
}

static ERTS_INLINE int
ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
			   ethr_dw_sint_t *new,
			   ethr_dw_sint_t *xchg)
{
    return ethr_dw_atomic_cmpxchg(var, new, xchg);
}

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#ifdef ETHR_SU_DW_NAINT_T__
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((erts_aint64_t) DW.dw_sint)
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
#else /* !ETHR_SU_DW_NAINT_T__ */
# 1470 "beam/erl_threads.h"
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32)		\
     | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD])		\
	& ((erts_aint64_t) 0xffffffff)))
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    do {								\
	DW.sint[ETHR_DW_SINT_LOW_WORD] =				\
	    (ethr_sint_t) (AINT64 & 0xffffffff);			\
	DW.sint[ETHR_DW_SINT_HIGH_WORD] = 				\
	    (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff);		\
    } while (0)
#endif /* !ETHR_SU_DW_NAINT_T__ */
# 1482 "beam/erl_threads.h"

#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, 			\
					AVarP, XchgVar, NewVar, 	\
					ModificationCode)		\
do {									\
    ethr_dw_sint_t dw_xchg__, dw_new__;					\
    ethr_dw_atomic_read(AVarP, &dw_xchg__);				\
    do {								\
	XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__);			\
	{								\
	    ModificationCode;						\
	}								\
	ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar);			\
    } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__));		\
} while (0)

#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER)				\
    									\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_init_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_set_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    ethr_dw_sint_t dw;							\
    ethr_dw_atomic_read_ ## BARRIER(var, &dw);				\
    return ERTS_DW_SINT_TO_AINT64__(dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg | val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg & val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = val);				\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp)			\
{									\
    ethr_dw_sint_t dw_xchg, dw_new;					\
    ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp);				\
    ERTS_AINT64_TO_DW_SINT__(dw_new, new);				\
    if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg))	\
	return exp;							\
    return ERTS_DW_SINT_TO_AINT64__(dw_xchg);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    {					\
					new = xchg & ~mask;		\
					new |= mask & set;		\
				    });					\
    return xchg;							\
}

ERTS_ATOMIC64_OPS_IMPL__(nob)
ERTS_ATOMIC64_OPS_IMPL__(mb)
ERTS_ATOMIC64_OPS_IMPL__(acqb)
ERTS_ATOMIC64_OPS_IMPL__(relb)
ERTS_ATOMIC64_OPS_IMPL__(ddrb)
ERTS_ATOMIC64_OPS_IMPL__(rb)
ERTS_ATOMIC64_OPS_IMPL__(wb)

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    ethr_dw_sint_t dw;
    ERTS_AINT64_TO_DW_SINT__(dw, val);
    sint[0] = dw.sint[0];
    sint[1] = dw.sint[1];
}

ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var)
{
    ethr_sint_t *sint;
    ethr_dw_sint_t dw;
    sint = ethr_dw_atomic_addr(var);
    dw.sint[0] = sint[0];
    dw.sint[1] = sint[1];
    return ERTS_DW_SINT_TO_AINT64__(dw);
}

#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1682 "beam/erl_threads.h"

#endif /* ARCH_32 */
# 1684 "beam/erl_threads.h"

#else /* !USE_THREADS */
# 1686 "beam/erl_threads.h"

/* Double word size atomics */

#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_atomic_init_nob erts_no_atomic_set
#define erts_atomic_set_nob erts_no_atomic_set
#define erts_atomic_read_nob erts_no_atomic_read
#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_atomic_inc_nob erts_no_atomic_inc
#define erts_atomic_dec_nob erts_no_atomic_dec
#define erts_atomic_add_read_nob erts_no_atomic_add_read
#define erts_atomic_add_nob erts_no_atomic_add
#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_atomic_read_band_nob erts_no_atomic_read_band
#define erts_atomic_xchg_nob erts_no_atomic_xchg
#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_atomic_init_mb erts_no_atomic_set
#define erts_atomic_set_mb erts_no_atomic_set
#define erts_atomic_read_mb erts_no_atomic_read
#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_atomic_inc_mb erts_no_atomic_inc
#define erts_atomic_dec_mb erts_no_atomic_dec
#define erts_atomic_add_read_mb erts_no_atomic_add_read
#define erts_atomic_add_mb erts_no_atomic_add
#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_atomic_read_band_mb erts_no_atomic_read_band
#define erts_atomic_xchg_mb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_atomic_init_acqb erts_no_atomic_set
#define erts_atomic_set_acqb erts_no_atomic_set
#define erts_atomic_read_acqb erts_no_atomic_read
#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_atomic_inc_acqb erts_no_atomic_inc
#define erts_atomic_dec_acqb erts_no_atomic_dec
#define erts_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_atomic_add_acqb erts_no_atomic_add
#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_atomic_init_relb erts_no_atomic_set
#define erts_atomic_set_relb erts_no_atomic_set
#define erts_atomic_read_relb erts_no_atomic_read
#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_atomic_inc_relb erts_no_atomic_inc
#define erts_atomic_dec_relb erts_no_atomic_dec
#define erts_atomic_add_read_relb erts_no_atomic_add_read
#define erts_atomic_add_relb erts_no_atomic_add
#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_atomic_read_band_relb erts_no_atomic_read_band
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_atomic_init_ddrb erts_no_atomic_set
#define erts_atomic_set_ddrb erts_no_atomic_set
#define erts_atomic_read_ddrb erts_no_atomic_read
#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_atomic_inc_ddrb erts_no_atomic_inc
#define erts_atomic_dec_ddrb erts_no_atomic_dec
#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_atomic_add_ddrb erts_no_atomic_add
#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
#define erts_atomic_read_rb erts_no_atomic_read
#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_atomic_inc_rb erts_no_atomic_inc
#define erts_atomic_dec_rb erts_no_atomic_dec
#define erts_atomic_add_read_rb erts_no_atomic_add_read
#define erts_atomic_add_rb erts_no_atomic_add
#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_atomic_read_band_rb erts_no_atomic_read_band
#define erts_atomic_xchg_rb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_atomic_init_wb erts_no_atomic_set
#define erts_atomic_set_wb erts_no_atomic_set
#define erts_atomic_read_wb erts_no_atomic_read
#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_atomic_inc_wb erts_no_atomic_inc
#define erts_atomic_dec_wb erts_no_atomic_dec
#define erts_atomic_add_read_wb erts_no_atomic_add_read
#define erts_atomic_add_wb erts_no_atomic_add
#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_atomic_read_band_wb erts_no_atomic_read_band
#define erts_atomic_xchg_wb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_atomic_set_dirty erts_no_atomic_set
#define erts_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_atomic32_init_nob erts_no_atomic32_set
#define erts_atomic32_set_nob erts_no_atomic32_set
#define erts_atomic32_read_nob erts_no_atomic32_read
#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_atomic32_inc_nob erts_no_atomic32_inc
#define erts_atomic32_dec_nob erts_no_atomic32_dec
#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_atomic32_add_nob erts_no_atomic32_add
#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_atomic32_init_mb erts_no_atomic32_set
#define erts_atomic32_set_mb erts_no_atomic32_set
#define erts_atomic32_read_mb erts_no_atomic32_read
#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_atomic32_inc_mb erts_no_atomic32_inc
#define erts_atomic32_dec_mb erts_no_atomic32_dec
#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_atomic32_add_mb erts_no_atomic32_add
#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_atomic32_init_acqb erts_no_atomic32_set
#define erts_atomic32_set_acqb erts_no_atomic32_set
#define erts_atomic32_read_acqb erts_no_atomic32_read
#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_atomic32_add_acqb erts_no_atomic32_add
#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_atomic32_init_relb erts_no_atomic32_set
#define erts_atomic32_set_relb erts_no_atomic32_set
#define erts_atomic32_read_relb erts_no_atomic32_read
#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_atomic32_inc_relb erts_no_atomic32_inc
#define erts_atomic32_dec_relb erts_no_atomic32_dec
#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_atomic32_add_relb erts_no_atomic32_add
#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_atomic32_init_ddrb erts_no_atomic32_set
#define erts_atomic32_set_ddrb erts_no_atomic32_set
#define erts_atomic32_read_ddrb erts_no_atomic32_read
#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_atomic32_add_ddrb erts_no_atomic32_add
#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
#define erts_atomic32_read_rb erts_no_atomic32_read
#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_atomic32_inc_rb erts_no_atomic32_inc
#define erts_atomic32_dec_rb erts_no_atomic32_dec
#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_atomic32_add_rb erts_no_atomic32_add
#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_atomic32_init_wb erts_no_atomic32_set
#define erts_atomic32_set_wb erts_no_atomic32_set
#define erts_atomic32_read_wb erts_no_atomic32_read
#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_atomic32_inc_wb erts_no_atomic32_inc
#define erts_atomic32_dec_wb erts_no_atomic32_dec
#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_atomic32_add_wb erts_no_atomic32_add
#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_atomic32_set_dirty erts_no_atomic32_set
#define erts_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_atomic64_init_nob erts_no_atomic64_set
#define erts_atomic64_set_nob erts_no_atomic64_set
#define erts_atomic64_read_nob erts_no_atomic64_read
#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_atomic64_inc_nob erts_no_atomic64_inc
#define erts_atomic64_dec_nob erts_no_atomic64_dec
#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_atomic64_add_nob erts_no_atomic64_add
#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_atomic64_init_mb erts_no_atomic64_set
#define erts_atomic64_set_mb erts_no_atomic64_set
#define erts_atomic64_read_mb erts_no_atomic64_read
#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_atomic64_inc_mb erts_no_atomic64_inc
#define erts_atomic64_dec_mb erts_no_atomic64_dec
#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_atomic64_add_mb erts_no_atomic64_add
#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_atomic64_init_acqb erts_no_atomic64_set
#define erts_atomic64_set_acqb erts_no_atomic64_set
#define erts_atomic64_read_acqb erts_no_atomic64_read
#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_atomic64_add_acqb erts_no_atomic64_add
#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_atomic64_init_relb erts_no_atomic64_set
#define erts_atomic64_set_relb erts_no_atomic64_set
#define erts_atomic64_read_relb erts_no_atomic64_read
#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_atomic64_inc_relb erts_no_atomic64_inc
#define erts_atomic64_dec_relb erts_no_atomic64_dec
#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_atomic64_add_relb erts_no_atomic64_add
#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_atomic64_init_ddrb erts_no_atomic64_set
#define erts_atomic64_set_ddrb erts_no_atomic64_set
#define erts_atomic64_read_ddrb erts_no_atomic64_read
#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_atomic64_add_ddrb erts_no_atomic64_add
#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_atomic64_init_rb erts_no_atomic64_set
#define erts_atomic64_set_rb erts_no_atomic64_set
#define erts_atomic64_read_rb erts_no_atomic64_read
#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_atomic64_inc_rb erts_no_atomic64_inc
#define erts_atomic64_dec_rb erts_no_atomic64_dec
#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_atomic64_add_rb erts_no_atomic64_add
#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_atomic64_init_wb erts_no_atomic64_set
#define erts_atomic64_set_wb erts_no_atomic64_set
#define erts_atomic64_read_wb erts_no_atomic64_read
#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_atomic64_inc_wb erts_no_atomic64_inc
#define erts_atomic64_dec_wb erts_no_atomic64_dec
#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_atomic64_add_wb erts_no_atomic64_add
#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_atomic64_set_dirty erts_no_atomic64_set
#define erts_atomic64_read_dirty erts_no_atomic64_read

#endif /* !USE_THREADS */
# 2058 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_thr_init(erts_thr_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_init(id);
    if (res)
	erts_thr_fatal_error(res, "initialize thread library");
#endif
# 2069 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_late_init(erts_thr_late_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_late_init(id);
    if (res)
	erts_thr_fatal_error(res, "complete initialization of thread library");
#endif
# 2079 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
		erts_thr_opts_t *opts)
{
#ifdef USE_THREADS
    int res = ethr_thr_create(tid, func, arg, opts);
    if (res)
	erts_thr_fatal_error(res, "create thread");
#endif
# 2090 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_join(erts_tid_t tid, void **thr_res)
{
#ifdef USE_THREADS
    int res = ethr_thr_join(tid, thr_res);
    if (res)
	erts_thr_fatal_error(res, "join thread");
#endif
# 2100 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_detach(erts_tid_t tid)
{
#ifdef USE_THREADS
    int res = ethr_thr_detach(tid);
    if (res)
	erts_thr_fatal_error(res, "detach thread");
#endif
# 2111 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_exit(void *res)
{
#ifdef USE_THREADS
    ethr_thr_exit(res);
    erts_thr_fatal_error(0, "terminate thread");
#endif
# 2121 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_install_exit_handler(void (*exit_handler)(void))
{
#ifdef USE_THREADS
    int res = ethr_install_exit_handler(exit_handler);
    if (res != 0)
	erts_thr_fatal_error(res, "install thread exit handler");
#endif
# 2131 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tid_t
erts_thr_self(void)
{
#ifdef USE_THREADS
    return ethr_self();
#else
# 2139 "beam/erl_threads.h"
    return 0;
#endif
# 2141 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
{
#ifdef USE_THREADS
    return ethr_getname(tid, buf, len);
#else
# 2149 "beam/erl_threads.h"
    return -1;
#endif
# 2151 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
erts_equal_tids(erts_tid_t x, erts_tid_t y)
{
#ifdef USE_THREADS
    return ethr_equal_tids(x, y);
#else
# 2160 "beam/erl_threads.h"
    return 1;
#endif
# 2162 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2174 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2180 "beam/erl_threads.h"
#endif
# 2181 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
		    int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2194 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
#endif
# 2200 "beam/erl_threads.h"
#endif
# 2201 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2214 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2220 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2224 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2227 "beam/erl_threads.h"
#endif
# 2228 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2240 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2243 "beam/erl_threads.h"
#endif
# 2244 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2256 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2259 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2263 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2266 "beam/erl_threads.h"
#endif
# 2267 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_destroy(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&mtx->lc);
#endif
# 2277 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&mtx->lcnt);
#endif
# 2280 "beam/erl_threads.h"
    res = ethr_mutex_destroy(&mtx->mtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy mutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2290 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy mutex");
    }
#endif
# 2293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2299 "beam/erl_threads.h"
erts_mtx_trylock(erts_mtx_t *mtx)
#endif
# 2301 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy(&mtx->lc))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2310 "beam/erl_threads.h"

    res = ethr_mutex_trylock(&mtx->mtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_x(res == 0, &mtx->lc,file,line);
#else
# 2317 "beam/erl_threads.h"
    erts_lc_trylock(res == 0, &mtx->lc);
#endif
# 2319 "beam/erl_threads.h"
#endif
# 2320 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, res);
#endif    
# 2323 "beam/erl_threads.h"
    return res;
#else
# 2325 "beam/erl_threads.h"
    return 0;
#endif
# 2327 "beam/erl_threads.h"

}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2334 "beam/erl_threads.h"
erts_mtx_lock(erts_mtx_t *mtx)
#endif
# 2336 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&mtx->lc, file, line);
#else
# 2342 "beam/erl_threads.h"
    erts_lc_lock(&mtx->lc);
#endif
# 2344 "beam/erl_threads.h"
#endif
# 2345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2348 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
#endif
# 2352 "beam/erl_threads.h"
#endif
# 2353 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_unlock(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2362 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2365 "beam/erl_threads.h"
    ethr_mutex_unlock(&mtx->mtx);
#endif
# 2367 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_mtx_is_locked(erts_mtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2379 "beam/erl_threads.h"
    return 0;
#endif
# 2381 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_init(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_init(cnd);
    if (res)
	erts_thr_fatal_error(res, "initialize condition variable");
#endif
# 2391 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_destroy(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_destroy(cnd);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy cond destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2407 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy condition variable");
    }
#endif
# 2410 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2420 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2423 "beam/erl_threads.h"
    res = ethr_cond_wait(cnd, &mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock(&mtx->lc);
#endif
# 2427 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2430 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post(&mtx->lcnt);
#endif
# 2433 "beam/erl_threads.h"
    if (res != 0 && res != EINTR)
	erts_thr_fatal_error(res, "wait on condition variable");
#endif
# 2436 "beam/erl_threads.h"
}

/*
 * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_cnd_signal(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_signal(cnd);
#endif
# 2454 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_cnd_broadcast(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_broadcast(cnd);
#endif
# 2463 "beam/erl_threads.h"
}

/* rwmutex */

ERTS_GLB_INLINE void
erts_rwmtx_set_reader_group(int no)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2475 "beam/erl_threads.h"
    res = ethr_rwmutex_set_reader_group(no);
    if (res != 0)
	erts_thr_fatal_error(res, "set reader group");
#endif
# 2479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
		      erts_rwmtx_opt_t *opt,
		      char *name,
		      Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
#endif
# 2494 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (name && name[0] == '\0')
      erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
    else
      erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
#endif
# 2500 "beam/erl_threads.h"
#endif
# 2501 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
		  char *name,
		  Eterm extra)
{
    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
		    erts_rwmtx_opt_t *opt,
		    char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2523 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
#endif
# 2526 "beam/erl_threads.h"
#endif
# 2527 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
{
    erts_rwmtx_init_opt(rwmtx, NULL, name);
}

ERTS_GLB_INLINE void
erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&rwmtx->lc);
#endif
# 2543 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&rwmtx->lcnt);
#endif
# 2546 "beam/erl_threads.h"
    res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwmutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2556 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwmutex");
    }
#endif
# 2559 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2565 "beam/erl_threads.h"
erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
#endif
# 2567 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2576 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2583 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2585 "beam/erl_threads.h"
#endif
# 2586 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
#endif
# 2589 "beam/erl_threads.h"
    
    return res;
#else
# 2592 "beam/erl_threads.h"
    return 0;
#endif
# 2594 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2600 "beam/erl_threads.h"
erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
#endif
# 2602 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2608 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2610 "beam/erl_threads.h"
#endif
# 2611 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2614 "beam/erl_threads.h"
    ethr_rwmutex_rlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2618 "beam/erl_threads.h"
#endif
# 2619 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2628 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2631 "beam/erl_threads.h"
    ethr_rwmutex_runlock(&rwmtx->rwmtx);
#endif
# 2633 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2640 "beam/erl_threads.h"
erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
#endif
# 2642 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2651 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2658 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2660 "beam/erl_threads.h"
#endif
# 2661 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2664 "beam/erl_threads.h"
    
    return res;
#else
# 2667 "beam/erl_threads.h"
    return 0;
#endif
# 2669 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2675 "beam/erl_threads.h"
erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
#endif
# 2677 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2683 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2685 "beam/erl_threads.h"
#endif
# 2686 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2689 "beam/erl_threads.h"
    ethr_rwmutex_rwlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2693 "beam/erl_threads.h"
#endif
# 2694 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2703 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2706 "beam/erl_threads.h"
    ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
#endif
# 2708 "beam/erl_threads.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_rwmtx_trywlock(erts_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wlock(erts_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx)
{

}

#endif
# 2735 "beam/erl_threads.h"

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2746 "beam/erl_threads.h"
    return 0;
#endif
# 2748 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2760 "beam/erl_threads.h"
    return 0;
#endif
# 2762 "beam/erl_threads.h"
}

/* No atomic ops */

ERTS_GLB_INLINE void
erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    var->sint[0] = val->sint[0];
    var->sint[1] = val->sint[1];
}

ERTS_GLB_INLINE void
erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    val->sint[0] = var->sint[0];
    val->sint[1] = var->sint[1];
}

ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *new_val,
					      erts_no_dw_atomic_t *old_val)
{
    if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
	erts_no_dw_atomic_read(var, old_val);
	return 0;
    }
    else {
	erts_no_dw_atomic_set(var, new_val);
	return !0;
    }
}

ERTS_GLB_INLINE void
erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read(erts_no_atomic_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_inc_read(erts_no_atomic_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_dec_read(erts_no_atomic_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic_inc(erts_no_atomic_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic_dec(erts_no_atomic_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
{
    erts_aint_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
		       erts_aint_t new,
		       erts_aint_t expected)
{
    erts_aint_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bset(erts_no_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    erts_aint_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic32 */

ERTS_GLB_INLINE void
erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read(erts_no_atomic32_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_inc(erts_no_atomic32_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_dec(erts_no_atomic32_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
{
    erts_aint32_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
			 erts_aint32_t new,
			 erts_aint32_t expected)
{
    erts_aint32_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    erts_aint32_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic64 */

ERTS_GLB_INLINE void
erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read(erts_no_atomic64_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_inc(erts_no_atomic64_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_dec(erts_no_atomic64_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
{
    erts_aint64_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
			 erts_aint64_t new,
			 erts_aint64_t expected)
{
    erts_aint64_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
			   erts_aint64_t mask,
			   erts_aint64_t set)
{
    erts_aint64_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* spinlock */

ERTS_GLB_INLINE void
erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3098 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
#endif
# 3101 "beam/erl_threads.h"
#else
# 3102 "beam/erl_threads.h"
    (void)lock;
#endif
# 3104 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
			 Uint16 opt)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3117 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
#endif
# 3120 "beam/erl_threads.h"
#else
# 3121 "beam/erl_threads.h"
    (void)lock;
#endif
# 3123 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_spinlock_init(erts_spinlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK);
#endif
# 3136 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
#endif
# 3139 "beam/erl_threads.h"
#else
# 3140 "beam/erl_threads.h"
    (void)lock;
#endif
# 3142 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_destroy(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3152 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3155 "beam/erl_threads.h"
    res = ethr_spinlock_destroy(&lock->slck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy spinlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3165 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3168 "beam/erl_threads.h"
    (void)lock;
#endif
# 3170 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spin_unlock(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&lock->lc);
#endif
# 3179 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&lock->lcnt);
#endif
# 3182 "beam/erl_threads.h"
    ethr_spin_unlock(&lock->slck);
#else
# 3184 "beam/erl_threads.h"
    (void)lock;
#endif
# 3186 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line)
#else
# 3192 "beam/erl_threads.h"
erts_spin_lock(erts_spinlock_t *lock)
#endif
# 3194 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&lock->lc,file,line);
#else
# 3200 "beam/erl_threads.h"
    erts_lc_lock(&lock->lc);
#endif
# 3202 "beam/erl_threads.h"
#endif
# 3203 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&lock->lcnt);
#endif
# 3206 "beam/erl_threads.h"
    ethr_spin_lock(&lock->slck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3210 "beam/erl_threads.h"
#else
# 3211 "beam/erl_threads.h"
    (void)lock;
#endif
# 3213 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3225 "beam/erl_threads.h"
    return 0;
#endif
# 3227 "beam/erl_threads.h"
}

/* rwspinlock */

ERTS_GLB_INLINE void
erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
#endif
# 3241 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
#endif
# 3244 "beam/erl_threads.h"
#else
# 3245 "beam/erl_threads.h"
    (void)lock;
#endif
# 3247 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_init(erts_rwlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK);
#endif
# 3259 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
#endif
# 3262 "beam/erl_threads.h"
#else
# 3263 "beam/erl_threads.h"
    (void)lock;
#endif
# 3265 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_destroy(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3275 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3278 "beam/erl_threads.h"
    res = ethr_rwlock_destroy(&lock->rwlck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3288 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3291 "beam/erl_threads.h"
    (void)lock;
#endif
# 3293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_read_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3302 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3305 "beam/erl_threads.h"
    ethr_read_unlock(&lock->rwlck);
#else
# 3307 "beam/erl_threads.h"
    (void)lock;
#endif
# 3309 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3315 "beam/erl_threads.h"
erts_read_lock(erts_rwlock_t *lock)
#endif
# 3317 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 3323 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3325 "beam/erl_threads.h"
#endif
# 3326 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3329 "beam/erl_threads.h"
    ethr_read_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3333 "beam/erl_threads.h"
#else
# 3334 "beam/erl_threads.h"
    (void)lock;
#endif
# 3336 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_write_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3348 "beam/erl_threads.h"
    ethr_write_unlock(&lock->rwlck);
#else
# 3350 "beam/erl_threads.h"
    (void)lock;
#endif
# 3352 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3358 "beam/erl_threads.h"
erts_write_lock(erts_rwlock_t *lock)
#endif
# 3360 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 3366 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3368 "beam/erl_threads.h"
#endif
# 3369 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3372 "beam/erl_threads.h"
    ethr_write_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3376 "beam/erl_threads.h"
#else
# 3377 "beam/erl_threads.h"
    (void)lock;
#endif
# 3379 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3391 "beam/erl_threads.h"
    return 0;
#endif
# 3393 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3405 "beam/erl_threads.h"
    return 0;
#endif
# 3407 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_create(keyp, keyname);
    if (res)
	erts_thr_fatal_error(res, "create thread specific data key");
#endif
# 3417 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_delete(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_delete(key);
    if (res)
	erts_thr_fatal_error(res, "delete thread specific data key");
#endif
# 3427 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_set(erts_tsd_key_t key, void *value)
{
#ifdef USE_THREADS
    int res = ethr_tsd_set(key, value);
    if (res)
	erts_thr_fatal_error(res, "set thread specific data");
#endif
# 3437 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void *
erts_tsd_get(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    return ethr_tsd_get(key);
#else
# 3445 "beam/erl_threads.h"
    return NULL;
#endif
# 3447 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
{
#ifdef USE_THREADS
    return (erts_tse_t *) ethr_get_ts_event();
#else
# 3454 "beam/erl_threads.h"
    return (erts_tse_t *) NULL;
#endif
# 3456 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_leave_ts_event(ep);
#endif
# 3463 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
{
#ifdef USE_THREADS
    int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
    if (res != 0)
	erts_thr_fatal_error(res, "prepare timed");
#endif
# 3472 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_set(&((ethr_ts_event *) ep)->event);
#endif
# 3479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_reset(&((ethr_ts_event *) ep)->event);
#endif
# 3486 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
#else
# 3493 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3495 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
{
#ifdef USE_THREADS
    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
#else
# 3502 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3504 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_twait(&((ethr_ts_event *) ep)->event,
			    (ethr_sint64_t) tmo);
#else
# 3512 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3514 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_stwait(&((ethr_ts_event *) ep)->event,
			     spincount,
			     (ethr_sint64_t) tmo);
#else
# 3523 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3525 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
#else
# 3532 "beam/erl_threads.h"
    return 0;
#endif
# 3534 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
{
#ifdef USE_THREADS
    int res = ethr_set_main_thr_status(on, no);
    if (res != 0)
	erts_thr_fatal_error(res, "set thread main status");
#endif
# 3543 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_thr_get_main_status(void)
{
#ifdef USE_THREADS
    int main_status;
    int res = ethr_get_main_thr_status(&main_status);
    if (res != 0)
	erts_thr_fatal_error(res, "get thread main status");
    return main_status;
#else
# 3554 "beam/erl_threads.h"
    return 1;
#endif
# 3556 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_yield(void)
{
#ifdef USE_THREADS
    int res = ETHR_YIELD();
    if (res != 0)
	erts_thr_fatal_error(res, "yield");
#endif    
# 3565 "beam/erl_threads.h"
}


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS

ERTS_GLB_INLINE void
erts_thr_kill(erts_tid_t tid, int sig) {
#ifdef USE_THREADS
  int res = ethr_kill((ethr_tid)tid, sig);
  if (res)
    erts_thr_fatal_error(res, "killing thread");
#endif
# 3577 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef USE_THREADS
    int res = ethr_sigmask(how, set, oset);
    if (res)
	erts_thr_fatal_error(res, "get or set signal mask");
#endif
# 3587 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef USE_THREADS
    int res;
    do {
	res = ethr_sigwait(set, sig);
    } while (res == EINTR);
    if (res)
	erts_thr_fatal_error(res, "to wait for signal");
#endif
# 3600 "beam/erl_threads.h"
}

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 3603 "beam/erl_threads.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 3605 "beam/erl_threads.h"

#endif /* #ifndef ERL_THREAD_H__ */
# 3607 "beam/erl_threads.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_atomic_init
#undef erts_atomic_set
#undef erts_atomic_read
#undef erts_atomic_inctest
#undef erts_atomic_dectest
#undef erts_atomic_inc
#undef erts_atomic_dec
#undef erts_atomic_addtest
#undef erts_atomic_add
#undef erts_atomic_xchg
#undef erts_atomic_cmpxchg
#undef erts_atomic_bor
#undef erts_atomic_band

#undef erts_atomic32_init
#undef erts_atomic32_set
#undef erts_atomic32_read
#undef erts_atomic32_inctest
#undef erts_atomic32_dectest
#undef erts_atomic32_inc
#undef erts_atomic32_dec
#undef erts_atomic32_addtest
#undef erts_atomic32_add
#undef erts_atomic32_xchg
#undef erts_atomic32_cmpxchg
#undef erts_atomic32_bor
#undef erts_atomic32_band

#endif
# 3641 "beam/erl_threads.h"
# 108 "beam/erl_thr_progress.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 108 "beam/erl_thr_progress.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 109 "beam/erl_thr_progress.h" 2

#ifdef ERTS_SMP

/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))

extern erts_tsd_key_t erts_thr_prgr_data_key__;

#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t

typedef struct {
    void *arg;
    void (*wakeup)(void *);
    void (*prepare_wait)(void *);
    void (*wait)(void *);
    void (*finalize_wait)(void *);
} ErtsThrPrgrCallbacks;

typedef struct {
    ERTS_THR_PRGR_ATOMIC current;
} ErtsThrPrgr;

typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)

extern ErtsThrPrgr erts_thr_prgr__;

void erts_thr_progress_pre_init(void);
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
					       ErtsThrPrgrCallbacks *,
					       int);
void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
			      ErtsThrPrgrVal value);
int erts_thr_progress_update(ErtsSchedulerData *esdp);
int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
void erts_thr_progress_unmanaged_continue__(int umrefc_ix);

void erts_thr_progress_dbg_print_state(void);

ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);

ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
#endif
# 168 "beam/erl_thr_progress.h"
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
					    ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
}

ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_managed;
}

ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)
{
    if (erts_thr_progress_is_managed_thread())
	return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
    else
	return erts_thr_progress_unmanaged_delay__();
}

ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
{
    ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
	   || erts_thr_progress_is_managed_thread());
    if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
	erts_thr_progress_unmanaged_continue__(handle);
}

#ifdef ERTS_ENABLE_LOCK_CHECK

ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_delaying;
}

#endif
# 233 "beam/erl_thr_progress.h"

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
{
    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
	return ((ErtsThrPrgrVal) 0);
    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
	return ((ErtsThrPrgrVal) 1);
    else
	return val + ((ErtsThrPrgrVal) 2);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData *esdp)
{
    ErtsThrPrgrData *tpd;
    ErtsThrPrgrVal val;
    if (esdp) {
	tpd = &esdp->thr_progress_data;
    managed_thread:
	val = tpd->confirmed;
	ERTS_THR_MEMORY_BARRIER;
    }
    else {
	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
	if (tpd && tpd->is_managed)
	    goto managed_thread;
	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
    }
    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
    return erts_thr_progress_current_to_later__(val);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
    else
	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}

ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
{
    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
	/* May have wrapped... */
	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
	    /*
	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
	     * these ranges 'current' is considered later than 'val0'.
	     */
	    return 1;
	}
    }
    return val1 > val0;
}

ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
{
    if (this == val)
	return 1;
    return erts_thr_progress_has_passed__(this, val);
}

ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
}

ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    if (val1 == val2)
	return 0;
    if (erts_thr_progress_has_passed__(val1, val2))
	return 1;
    else
	return -1;
}	

ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)
{
    ErtsThrPrgrVal current = erts_thr_progress_current();
    return erts_thr_progress_has_reached_this(current, val);
}

#endif
# 326 "beam/erl_thr_progress.h"

#endif /* ERTS_SMP */
# 328 "beam/erl_thr_progress.h"

#endif
# 330 "beam/erl_thr_progress.h"
# 33 "beam/erl_port.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/erl_port.h"
# 1 "beam/erl_trace.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2012-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef ERL_TRACE_H__FLAGS__
#define ERL_TRACE_H__FLAGS__
/*
 * NOTE! The bits used for these flags matter. The flag with
 * the least significant bit will take precedence!
 *
 * The "now timestamp" has highest precedence due to
 * compatibility reasons.
 */
#define ERTS_TRACE_FLG_NOW_TIMESTAMP			(1 << 0)
#define ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP	(1 << 1)
#define ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP		(1 << 2)

/*
 * The bits used effects trace flags (of processes and ports)
 * as well as sequential trace flags. If changed make sure
 * these arn't messed up...
 */
#define ERTS_TRACE_TS_TYPE_BITS 			3
#define ERTS_TRACE_TS_TYPE_MASK					\
    ((1 << ERTS_TRACE_TS_TYPE_BITS) - 1)

#define ERTS_TFLGS2TSTYPE(TFLGS)				\
    ((int) (((TFLGS) >> ERTS_TRACE_FLAGS_TS_TYPE_SHIFT) 	\
	    & ERTS_TRACE_TS_TYPE_MASK))
#define ERTS_SEQTFLGS2TSTYPE(SEQTFLGS)				\
    ((int) (((SEQTFLGS) >> ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)	\
	    & ERTS_TRACE_TS_TYPE_MASK))

#endif /* ERL_TRACE_H__FLAGS__ */
# 51 "beam/erl_trace.h"

#if !defined(ERL_TRACE_H__) && !defined(ERTS_ONLY_INCLUDE_TRACE_FLAGS)
#define ERL_TRACE_H__

struct binary;

/* erl_bif_trace.c */
Eterm erl_seq_trace_info(Process *p, Eterm arg1);
void erts_system_monitor_clear(Process *c_p);
void erts_system_profile_clear(Process *c_p);

/* erl_trace.c */
void erts_init_trace(void);
void erts_trace_check_exiting(Eterm exiting);
Eterm erts_set_system_seq_tracer(Process *c_p,
				 ErtsProcLocks c_p_locks,
				 Eterm new);
Eterm erts_get_system_seq_tracer(void);
void erts_change_default_tracing(int setflags, Uint *flagsp, Eterm *tracerp);
void erts_get_default_tracing(Uint *flagsp, Eterm *tracerp);
void erts_set_system_monitor(Eterm monitor);
Eterm erts_get_system_monitor(void);
int erts_is_tracer_proc_valid(Process* p);

#ifdef ERTS_SMP
void erts_check_my_tracer_proc(Process *);
void erts_block_sys_msg_dispatcher(void);
void erts_release_sys_msg_dispatcher(void);
void erts_foreach_sys_msg_in_q(void (*func)(Eterm,
					    Eterm,
					    Eterm,
					    ErlHeapFragment *));
void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *);
#endif
# 85 "beam/erl_trace.h"

void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *);
void trace_send(Process*, Eterm, Eterm);
void trace_receive(Process*, Eterm);
Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, Eterm* args,
		       int local, Eterm *tracer_pid);
void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, Eterm *tracer_pid);
void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value,
			  Eterm *tracer);
void erts_trace_return_to(Process *p, BeamInstr *pc);
void trace_sched(Process*, Eterm);
void trace_proc(Process*, Process*, Eterm, Eterm);
void trace_proc_spawn(Process*, Eterm pid, Eterm mod, Eterm func, Eterm args);
void save_calls(Process *p, Export *);
void trace_gc(Process *p, Eterm what);
/* port tracing */
void trace_virtual_sched(Process*, Eterm);
void trace_sched_ports(Port *pp, Eterm);
void trace_sched_ports_where(Port *pp, Eterm, Eterm);
void trace_port(Port *, Eterm what, Eterm data);
void trace_port_open(Port *, Eterm calling_pid, Eterm drv_name);

/* system_profile */
void erts_set_system_profile(Eterm profile);
Eterm erts_get_system_profile(void);
void profile_scheduler(Eterm scheduler_id, Eterm);
void profile_scheduler_q(Eterm scheduler_id, Eterm state, Eterm no_schedulers, Uint Ms, Uint s, Uint us);
void profile_runnable_proc(Process* p, Eterm status);
void profile_runnable_port(Port* p, Eterm status);
void erts_system_profile_setup_active_schedulers(void);

/* system_monitor */
void monitor_long_gc(Process *p, Uint time);
void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time);
void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time);
void monitor_large_heap(Process *p);
void monitor_generic(Process *p, Eterm type, Eterm spec);
Uint erts_trace_flag2bit(Eterm flag);
int erts_trace_flags(Eterm List, 
		 Uint *pMask, Eterm *pTracer, int *pCpuTimestamp);
Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);

#ifdef ERTS_SMP
void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)				\
do {									\
    if ((ESDP)->pending_trace_msgs)					\
	erts_send_pending_trace_msgs((ESDP));				\
} while (0)
#else
# 135 "beam/erl_trace.h"
#define ERTS_SMP_CHK_PEND_TRACE_MSGS(ESDP)
#endif
# 137 "beam/erl_trace.h"

#define seq_trace_output(token, msg, type, receiver, process) \
seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL)
#define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \
seq_trace_output_generic((token), (msg), (type), (receiver), NULL, (exitfrom))
void seq_trace_output_generic(Eterm token, Eterm msg, Uint type, 
			      Eterm receiver, Process *process, Eterm exitfrom);

int seq_trace_update_send(Process *process);

Eterm erts_seq_trace(Process *process, 
		     Eterm atom_type, Eterm atom_true_or_false, 
		     int build_result);

struct trace_pattern_flags {
    unsigned int breakpoint : 1; /* Set if any other is set */
    unsigned int local      : 1; /* Local call trace breakpoint */
    unsigned int meta       : 1; /* Metadata trace breakpoint */
    unsigned int call_count : 1; /* Fast call count breakpoint */
    unsigned int call_time  : 1; /* Fast call time breakpoint */
};
extern const struct trace_pattern_flags erts_trace_pattern_flags_off;
extern int erts_call_time_breakpoint_tracing;
int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified,
			   struct binary* match_prog_set,
			   struct binary *meta_match_prog_set,
			   int on, struct trace_pattern_flags,
			   Eterm meta_tracer_pid, int is_blocking);
void
erts_get_default_trace_pattern(int *trace_pattern_is_on,
			       struct binary **match_spec,
			       struct binary **meta_match_spec,
			       struct trace_pattern_flags *trace_pattern_flags,
			       Eterm *meta_tracer_pid);
int erts_is_default_trace_enabled(void);
void erts_bif_trace_init(void);
int erts_finish_breakpointing(void);

#endif /* ERL_TRACE_H__ */
# 176 "beam/erl_trace.h"
# 34 "beam/erl_port.h" 2

#ifndef __WIN32__
#define ERTS_DEFAULT_MAX_PORTS (1 << 16)
#else
# 38 "beam/erl_port.h"
/* 
 *  Do not default to as many max ports on Windows
 *  as there are no os limits to stop system
 *  from running amok. If allowed to go too high
 *  windows rarely recovers from the errors and
 *  other OS processes can be effected. 
 */
#define ERTS_DEFAULT_MAX_PORTS (1 << 13)
#endif /* __WIN32__ */
# 47 "beam/erl_port.h"
#define ERTS_MIN_PORTS 1024

extern int erts_port_synchronous_ops;
extern int erts_port_schedule_all_ops;
extern int erts_port_parallelism;

typedef struct erts_driver_t_ erts_driver_t;

/*
 * It would have been preferred to use NULL as value of
 * ERTS_INVALID_ERL_DRV_PORT. That would, however, not be
 * backward compatible. In pre-R16 systems, 0 was a valid
 * port handle and -1 was used as invalid handle, so we
 * are stuck with it.
 */
#define ERTS_INVALID_ERL_DRV_PORT ((struct _erl_drv_port *) ((SWord) -1))
#ifdef DEBUG
/* Make sure we use this api, and do not cast directly */
#define ERTS_ErlDrvPort2Port(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((Port *) ((PH) - 4711)))
#define ERTS_Port2ErlDrvPort(PH)	\
    ((PH) == ERTS_INVALID_ERL_DRV_PORT	\
     ? ERTS_INVALID_ERL_DRV_PORT	\
     : ((ErlDrvPort) ((PH) + 4711)))
#else
# 74 "beam/erl_port.h"
#define ERTS_ErlDrvPort2Port(PH) ((Port *) (PH))
#define ERTS_Port2ErlDrvPort(PH) ((ErlDrvPort) (PH))
#endif
# 77 "beam/erl_port.h"

#define SMALL_IO_QUEUE 5   /* Number of fixed elements */

typedef struct {
    ErlDrvSizeT size;       /* total size in bytes */

    SysIOVec* v_start;
    SysIOVec* v_end;
    SysIOVec* v_head;
    SysIOVec* v_tail;
    SysIOVec  v_small[SMALL_IO_QUEUE];

    ErlDrvBinary** b_start;
    ErlDrvBinary** b_end;
    ErlDrvBinary** b_head;
    ErlDrvBinary** b_tail;
    ErlDrvBinary*  b_small[SMALL_IO_QUEUE];
} ErlIOQueue;

typedef struct line_buf {  /* Buffer used in line oriented I/O */
    ErlDrvSizeT bufsiz;      /* Size of character buffer */
    ErlDrvSizeT ovlen;       /* Length of overflow data */
    ErlDrvSizeT ovsiz;       /* Actual size of overflow buffer */
    char data[1];            /* Starting point of buffer data,
			      data[0] is a flag indicating an unprocess CR,
			      The rest is the overflow buffer. */
} LineBuf;

/*
 * Items part of erlang:port_info/1 result. Note am_registered_name
 * *need* to be first.
 */

#define ERTS_PORT_INFO_1_ITEMS				\
    {	am_registered_name,	/* Needs to be first */	\
	am_name,					\
	am_links,					\
	am_id,						\
	am_connected,					\
	am_input,					\
	am_output,					\
	am_os_pid }

/*
 * Port Specific Data.
 *
 * Only use PrtSD for very rarely used data.
 */

#define ERTS_PRTSD_SCHED_ID 0

#define ERTS_PRTSD_SIZE 1

typedef struct {
    void *data[ERTS_PRTSD_SIZE];
} ErtsPrtSD;

#ifdef ERTS_SMP
typedef struct ErtsXPortsList_ ErtsXPortsList;
#endif
# 137 "beam/erl_port.h"

/*
 * Port locking:
 *
 * Locking is done either driver specific or port specific. When
 * driver specific locking is used, all instances of the driver,
 * i.e. ports running the driver, share the same lock. When port
 * specific locking is used each instance have its own lock.
 *
 * Most fields in the Port structure are protected by the lock
 * referred to by the 'lock' field. This lock is shared between
 * all ports running the same driver when driver specific locking
 * is used.
 *
 * The 'sched' field is protected by the run queue lock that the
 * port currently is assigned to.
 *
 */

struct _erl_drv_port {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    ErtsPortTaskSched sched;
    ErtsPortTaskHandle timeout_task;
#ifdef ERTS_SMP
    erts_mtx_t *lock;
    ErtsXPortsList *xports;
    erts_smp_atomic_t run_queue;
#else
# 166 "beam/erl_port.h"
    erts_atomic32_t refc;
    int cleanup;
#endif
# 169 "beam/erl_port.h"
    erts_atomic_t connected;	/* A connected process */
    Eterm caller;		/* Current caller. */
    erts_smp_atomic_t data;	/* Data associated with port. */
    Uint bytes_in;		/* Number of bytes read */
    Uint bytes_out;		/* Number of bytes written */

    ErlIOQueue ioq;              /* driver accessible i/o queue */
    DistEntry *dist_entry;       /* Dist entry used in DISTRIBUTION */
    char *name;		         /* String used in the open */
    erts_driver_t* drv_ptr;
    UWord drv_data;
    SWord os_pid;                /* Child process ID */
    ErtsProcList *suspended;	 /* List of suspended processes. */
    LineBuf *linebuf;            /* Buffer to hold data not ready for
				    process to get (line oriented I/O)*/
    erts_atomic32_t state;	 /* Status and type flags */
    int control_flags;		 /* Flags for port_control()  */
    ErlDrvPDL port_data_lock;

    ErtsPrtSD *psd;		 /* Port specific data */
    int reds; /* Only used while executing driver callbacks */
};


void erts_init_port_data(Port *);
void erts_cleanup_port_data(Port *);
Uint erts_port_data_size(Port *);
ErlOffHeap *erts_port_data_offheap(Port *);

#define ERTS_PORT_GET_CONNECTED(PRT) \
    ((Eterm) erts_atomic_read_nob(&(PRT)->connected))
#define ERTS_PORT_SET_CONNECTED(PRT, PID) \
    erts_atomic_set_relb(&(PRT)->connected, (erts_aint_t) (PID))
#define ERTS_PORT_INIT_CONNECTED(PRT, PID) \
    erts_atomic_init_nob(&(PRT)->connected, (erts_aint_t) (PID))


struct erl_drv_port_data_lock {
    erts_mtx_t mtx;
    erts_atomic_t refc;
    Port *prt;
};

ERTS_GLB_INLINE ErtsRunQueue *erts_port_runq(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsRunQueue *
erts_port_runq(Port *prt)
{
#ifdef ERTS_SMP
    ErtsRunQueue *rq1, *rq2;
    rq1 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
    if (!rq1)
	return NULL;
    while (1) {
	erts_smp_runq_lock(rq1);
	rq2 = (ErtsRunQueue *) erts_smp_atomic_read_nob(&prt->run_queue);
	if (rq1 == rq2)
	    return rq1;
	erts_smp_runq_unlock(rq1);
	rq1 = rq2;
	if (!rq1)
	    return NULL;
    }
#else
# 235 "beam/erl_port.h"
    return ERTS_RUNQ_IX(0);
#endif
# 237 "beam/erl_port.h"
}

#endif
# 240 "beam/erl_port.h"


ERTS_GLB_INLINE void *erts_prtsd_get(Port *p, int ix);
ERTS_GLB_INLINE void *erts_prtsd_set(Port *p, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_prtsd_get(Port *prt, int ix)
{
    return prt->psd ? prt->psd->data[ix] : NULL;
}

ERTS_GLB_INLINE void *
erts_prtsd_set(Port *prt, int ix, void *data)
{
    if (prt->psd) {
	void *old = prt->psd->data[ix];
	prt->psd->data[ix] = data;
	return old;
    }
    else {
	prt->psd = erts_alloc(ERTS_ALC_T_PRTSD, sizeof(ErtsPrtSD));
	prt->psd->data[ix] = data;
	return NULL;
    }
}

#endif
# 269 "beam/erl_port.h"

Eterm erts_request_io_bytes(Process *c_p);


/* port status flags */

#define ERTS_PORT_SFLG_CONNECTED	((Uint32) (1 <<  0))
/* Port have begun exiting */
#define ERTS_PORT_SFLG_EXITING		((Uint32) (1 <<  1))
/* Distribution port */
#define ERTS_PORT_SFLG_DISTRIBUTION	((Uint32) (1 <<  2))
#define ERTS_PORT_SFLG_BINARY_IO	((Uint32) (1 <<  3))
#define ERTS_PORT_SFLG_SOFT_EOF		((Uint32) (1 <<  4))
/* Flow control */
/* Port is closing (no i/o accepted) */
#define ERTS_PORT_SFLG_CLOSING		((Uint32) (1 <<  5))
/* Send a closed message when terminating */
#define ERTS_PORT_SFLG_SEND_CLOSED	((Uint32) (1 <<  6))
/* Line orinted io on port */  
#define ERTS_PORT_SFLG_LINEBUF_IO	((Uint32) (1 <<  7))
/* Immortal port (only certain system ports) */
#define ERTS_PORT_SFLG_FREE		((Uint32) (1 <<  8))
#define ERTS_PORT_SFLG_INITIALIZING	((Uint32) (1 <<  9))
/* Port uses port specific locking (opposed to driver specific locking) */
#define ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK ((Uint32) (1 << 10))
#define ERTS_PORT_SFLG_INVALID		((Uint32) (1 << 11))
/* Last port to terminate halts the emulator */
#define ERTS_PORT_SFLG_HALT		((Uint32) (1 << 12))
#ifdef DEBUG
/* Only debug: make sure all flags aren't cleared unintentionally */
#define ERTS_PORT_SFLG_PORT_DEBUG	((Uint32) (1 << 31))
#endif
# 301 "beam/erl_port.h"

/* Combinations of port status flags */ 
#define ERTS_PORT_SFLGS_DEAD						\
  (ERTS_PORT_SFLG_FREE | ERTS_PORT_SFLG_INITIALIZING)
#define ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
  (ERTS_PORT_SFLGS_DEAD | ERTS_PORT_SFLG_INVALID)
#define ERTS_PORT_SFLGS_INVALID_LOOKUP					\
  (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP				\
   | ERTS_PORT_SFLG_EXITING						\
   | ERTS_PORT_SFLG_CLOSING)
#define ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP				\
  (ERTS_PORT_SFLGS_INVALID_LOOKUP					\
   | ERTS_PORT_SFLG_DISTRIBUTION)

/*
 * Costs in reductions for some port operations.
 */
#define ERTS_PORT_REDS_EXECUTE		(CONTEXT_REDS/4)
#define ERTS_PORT_REDS_FREE		(CONTEXT_REDS/400)
#define ERTS_PORT_REDS_TIMEOUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_INPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_OUTPUT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EVENT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUTV	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CMD_OUTPUT	(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_EXIT		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CONNECT		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_UNLINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_LINK		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_BADSIG		(CONTEXT_REDS/200)
#define ERTS_PORT_REDS_CONTROL		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_CALL		(CONTEXT_REDS/50)
#define ERTS_PORT_REDS_INFO		(CONTEXT_REDS/100)
#define ERTS_PORT_REDS_TERMINATE	(CONTEXT_REDS/50)

void print_port_info(Port *, int, void *);
void erts_port_free(Port *);
#ifndef ERTS_SMP
void erts_port_cleanup(Port *);
#endif
# 341 "beam/erl_port.h"
void erts_fire_port_monitor(Port *prt, Eterm ref);
#ifdef ERTS_SMP
int erts_port_handle_xports(Port *);
#endif
# 345 "beam/erl_port.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_lc_is_port_locked(Port *);
#endif
# 349 "beam/erl_port.h"

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt);
ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc);
ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt);

ERTS_GLB_INLINE int erts_smp_port_trylock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_lock(Port *prt);
ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void erts_port_inc_refc(Port *prt)
{
    erts_ptab_atmc_inc_refc(&prt->common);
}

ERTS_GLB_INLINE void erts_port_dec_refc(Port *prt)
{
    int referred = erts_ptab_atmc_dec_test_refc(&prt->common);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE void erts_port_add_refc(Port *prt, Sint32 add_refc)
{
    int referred = erts_ptab_atmc_add_test_refc(&prt->common, add_refc);
    if (!referred)
	erts_port_free(prt);
}

ERTS_GLB_INLINE Sint erts_port_read_refc(Port *prt)
{
    return erts_ptab_atmc_read_refc(&prt->common);
}

ERTS_GLB_INLINE int
erts_smp_port_trylock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    return erts_mtx_trylock(prt->lock);
#else
# 393 "beam/erl_port.h"
    return 0;
#endif
# 395 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_lock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_lock(prt->lock);
#endif
# 405 "beam/erl_port.h"
}

ERTS_GLB_INLINE void
erts_smp_port_unlock(Port *prt)
{
#ifdef ERTS_SMP
    /* *Need* to be a managed thread */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
    erts_mtx_unlock(prt->lock);
#endif
# 415 "beam/erl_port.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 418 "beam/erl_port.h"


#define ERTS_INVALID_PORT_OPT(PP, ID, FLGS)			\
    (!(PP)							\
     || (erts_atomic32_read_nob(&(PP)->state) & (FLGS))	\
     || (PP)->common.id != (ID))

/* port lookup */

#define INVALID_PORT(PP, ID) \
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_LOOKUP)

/* Invalidate trace port if anything suspicious, for instance
 * that the port is a distribution port or it is busy.
 */
#define INVALID_TRACER_PORT(PP, ID)					\
  ERTS_INVALID_PORT_OPT((PP), (ID), ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP)

#define ERTS_PORT_SCHED_ID(P, ID) \
  ((Uint) (UWord) erts_prtsd_set((P), ERTS_PSD_SCHED_ID, (void *) (UWord) (ID)))

extern const Port erts_invalid_port;
#define ERTS_PORT_LOCK_BUSY ((Port *) &erts_invalid_port)

int erts_is_port_ioq_empty(Port *);
void erts_terminate_port(Port *);

#ifdef ERTS_SMP
Port *erts_de2port(DistEntry *, Process *, ErtsProcLocks);
#endif
# 448 "beam/erl_port.h"

ERTS_GLB_INLINE Port *erts_pix2port(int);
ERTS_GLB_INLINE Port *erts_port_lookup_raw(Eterm);
ERTS_GLB_INLINE Port *erts_port_lookup(Eterm, Uint32);
ERTS_GLB_INLINE Port*erts_id2port(Eterm id);
ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32);
ERTS_GLB_INLINE void erts_port_release(Port *);
#ifdef ERTS_SMP
ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs);
ERTS_GLB_INLINE void erts_thr_port_release(Port *prt);
#endif
# 459 "beam/erl_port.h"
ERTS_GLB_INLINE Port *erts_thr_drvport2port(ErlDrvPort, int);
ERTS_GLB_INLINE Port *erts_drvport2port_state(ErlDrvPort, erts_aint32_t *);
ERTS_GLB_INLINE Eterm erts_drvport2id(ErlDrvPort);
ERTS_GLB_INLINE Uint32 erts_portid2status(Eterm);
ERTS_GLB_INLINE int erts_is_port_alive(Eterm);
ERTS_GLB_INLINE int erts_is_valid_tracer_port(Eterm);
ERTS_GLB_INLINE int erts_port_driver_callback_epilogue(Port *, erts_aint32_t *);

#define erts_drvport2port(Prt) erts_drvport2port_state((Prt), NULL)

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Port *erts_pix2port(int ix)
{
    Port *prt;
    ASSERT(0 <= ix && ix < erts_ptab_max(&erts_port));
    prt = (Port *) erts_ptab_pix2intptr_nob(&erts_port, ix);
    return prt == ERTS_PORT_LOCK_BUSY ? NULL : prt;
}

ERTS_GLB_INLINE Port *
erts_port_lookup_raw(Eterm id)
{
    Port *prt;

    ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));
    return prt && prt->common.id == id ? prt : NULL;
}

ERTS_GLB_INLINE Port *
erts_port_lookup(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt = erts_port_lookup_raw(id);
    return (!prt
	    ? NULL
	    : ((invalid_sflgs & erts_atomic32_read_nob(&prt->state))
	       ? NULL
	       : prt));
}


ERTS_GLB_INLINE Port*
erts_id2port(Eterm id)
{
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

    erts_smp_port_lock(prt);
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_LOOKUP) {
	erts_smp_port_unlock(prt);
	return NULL;
    }

    return prt;
}


ERTS_GLB_INLINE Port*
erts_id2port_sflgs(Eterm id,
		   Process *c_p, ErtsProcLocks c_p_locks,
		   Uint32 invalid_sflgs)
{
#ifdef ERTS_SMP
    int no_proc_locks = !c_p || !c_p_locks;
#endif
# 543 "beam/erl_port.h"
    erts_aint32_t state;
    Port *prt;

    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());

    if (is_not_internal_port(id))
	return NULL;

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id)
	return NULL;

#ifdef ERTS_SMP
    if (no_proc_locks)
	erts_smp_port_lock(prt);
    else if (erts_smp_port_trylock(prt) == EBUSY) {
	/* Unlock process locks, and acquire locks in lock order... */
	erts_smp_proc_unlock(c_p, c_p_locks);
	erts_smp_port_lock(prt);
	erts_smp_proc_lock(c_p, c_p_locks);
    }
#endif
# 568 "beam/erl_port.h"
    state = erts_atomic32_read_nob(&prt->state);
    if (state & invalid_sflgs) {
#ifdef ERTS_SMP
	erts_smp_port_unlock(prt);
#endif
# 573 "beam/erl_port.h"
	return NULL;
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_port_release(Port *prt)
{
    /* Only allowed to be called from managed threads */
    ERTS_SMP_LC_ASSERT(erts_thr_progress_is_managed_thread());
#ifdef ERTS_SMP
    erts_smp_port_unlock(prt);
#else
# 587 "beam/erl_port.h"
    if (prt->cleanup) {
	prt->cleanup = 0;
	erts_port_cleanup(prt);
    }
#endif
# 592 "beam/erl_port.h"
}

#ifdef ERTS_SMP

/*
 * erts_thr_id2port_sflgs() and erts_thr_port_release() can
 * be used by unmanaged threads in the SMP case.
 */
ERTS_GLB_INLINE Port *
erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs)
{
    Port *prt;
    ErtsThrPrgrDelayHandle dhndl;

    if (is_not_internal_port(id))
	return NULL;

    dhndl = erts_thr_progress_unmanaged_delay();

    prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port,
					     internal_port_index(id));

    if (!prt || prt->common.id != id) {
	erts_thr_progress_unmanaged_continue(dhndl);
	prt = NULL;
    }
    else {
	erts_aint32_t state;
	if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) {
	    erts_port_inc_refc(prt);
	    erts_thr_progress_unmanaged_continue(dhndl);
	}

	erts_mtx_lock(prt->lock);
	state = erts_atomic32_read_nob(&prt->state);
	if (state & invalid_sflgs) {
	    erts_mtx_unlock(prt->lock);
	    if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
		erts_port_dec_refc(prt);
	    prt = NULL;
	}
    }

    return prt;
}

ERTS_GLB_INLINE void
erts_thr_port_release(Port *prt)
{
    erts_mtx_unlock(prt->lock);
#ifdef ERTS_SMP
    if (!erts_thr_progress_is_managed_thread())
	erts_port_dec_refc(prt);
#endif
# 646 "beam/erl_port.h"
}

#endif
# 649 "beam/erl_port.h"

ERTS_GLB_INLINE Port *
erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    ASSERT(prt != NULL);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;

    if (lock_pdl && prt->port_data_lock)
	driver_pdl_lock(prt->port_data_lock);

#if ERTS_ENABLE_LOCK_CHECK
    if (!ERTS_IS_CRASH_DUMPING) {
	if (erts_lc_is_emu_thr()) {
	    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	    ERTS_LC_ASSERT(!prt->port_data_lock
			   || erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
	else {
	    ERTS_LC_ASSERT(prt->port_data_lock);
	    ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&prt->port_data_lock->mtx));
	}
    }
#endif
# 674 "beam/erl_port.h"

    if (erts_atomic32_read_nob(&prt->state)
	& ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) {
	if (lock_pdl && prt->port_data_lock)
	    driver_pdl_unlock(prt->port_data_lock);
	return ERTS_INVALID_ERL_DRV_PORT;
    }
    return prt;
}

ERTS_GLB_INLINE Port *
erts_drvport2port_state(ErlDrvPort drvport, erts_aint32_t *statep)
{
    Port *prt = ERTS_ErlDrvPort2Port(drvport);
    erts_aint32_t state;
    ASSERT(prt);
    ERTS_LC_ASSERT(erts_lc_is_emu_thr());
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return ERTS_INVALID_ERL_DRV_PORT;
    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)
		       || ERTS_IS_CRASH_DUMPING);
    /* 
     * This state check is only needed since a driver callback
     * might terminate the port, and then call back into the
     * emulator. Drivers should preferably have been forbidden
     * to call into the emulator after terminating the port,
     * but it has been like this for ages. Perhaps forbid this
     * in some future major release?
     */
    state = erts_atomic32_read_nob(&prt->state);
    if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP)
	return ERTS_INVALID_ERL_DRV_PORT;
    if (statep)
	*statep = state;
    return prt;
}

ERTS_GLB_INLINE Eterm
erts_drvport2id(ErlDrvPort drvport)
{
    Port *prt = erts_drvport2port(drvport);
    if (prt == ERTS_INVALID_ERL_DRV_PORT)
	return am_undefined;
    else
	return prt->common.id;
}

ERTS_GLB_INLINE Uint32
erts_portid2status(Eterm id)
{
    Port *prt = erts_port_lookup_raw(id);
    if (prt)
	return (Uint32) erts_atomic32_read_acqb(&prt->state);
    else
	return ERTS_PORT_SFLG_INVALID;
}

ERTS_GLB_INLINE int
erts_is_port_alive(Eterm id)
{
    return !(erts_portid2status(id) & (ERTS_PORT_SFLG_INVALID
				       | ERTS_PORT_SFLGS_DEAD));
}

ERTS_GLB_INLINE int
erts_is_valid_tracer_port(Eterm id)
{
    return !(erts_portid2status(id) & ERTS_PORT_SFLGS_INVALID_TRACER_LOOKUP);
}

ERTS_GLB_INLINE int
erts_port_driver_callback_epilogue(Port *prt, erts_aint32_t *statep)
{
    int reds = 0;
    erts_aint32_t state;

    ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));

    state = erts_atomic32_read_nob(&prt->state);
    if ((state & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(prt)) {
	reds += ERTS_PORT_REDS_TERMINATE;
	erts_terminate_port(prt);
	state = erts_atomic32_read_nob(&prt->state);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
    }

#ifdef ERTS_SMP
    if (prt->xports) {
	reds += erts_port_handle_xports(prt);
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	ASSERT(!prt->xports);
    }
#endif
# 767 "beam/erl_port.h"

    if (statep)
	*statep = state;

    return reds;
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 775 "beam/erl_port.h"

void erts_port_resume_procs(Port *);

struct binary;

#define ERTS_P2P_SIG_TYPE_BAD			0
#define ERTS_P2P_SIG_TYPE_OUTPUT		1
#define ERTS_P2P_SIG_TYPE_OUTPUTV		2
#define ERTS_P2P_SIG_TYPE_CONNECT		3
#define ERTS_P2P_SIG_TYPE_EXIT			4
#define ERTS_P2P_SIG_TYPE_CONTROL		5
#define ERTS_P2P_SIG_TYPE_CALL			6
#define ERTS_P2P_SIG_TYPE_INFO			7
#define ERTS_P2P_SIG_TYPE_LINK			8
#define ERTS_P2P_SIG_TYPE_UNLINK		9

#define ERTS_P2P_SIG_TYPE_BITS			4
#define ERTS_P2P_SIG_TYPE_MASK \
    ((1 << ERTS_P2P_SIG_TYPE_BITS) - 1)

#define ERTS_P2P_SIG_DATA_FLG(N) \
    (1 << (ERTS_P2P_SIG_TYPE_BITS + (N)))
#define ERTS_P2P_SIG_DATA_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG(0)
#define ERTS_P2P_SIG_DATA_FLG_REPLY		ERTS_P2P_SIG_DATA_FLG(1)
#define ERTS_P2P_SIG_DATA_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG(2)
#define ERTS_P2P_SIG_DATA_FLG_FORCE		ERTS_P2P_SIG_DATA_FLG(3)
#define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT	ERTS_P2P_SIG_DATA_FLG(4)
#define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK	ERTS_P2P_SIG_DATA_FLG(5)
#define ERTS_P2P_SIG_DATA_FLG_SCHED		ERTS_P2P_SIG_DATA_FLG(6)
#define ERTS_P2P_SIG_DATA_FLG_ASYNC		ERTS_P2P_SIG_DATA_FLG(7)

struct ErtsProc2PortSigData_ {
    int flags;
    Eterm caller;
    Uint32 ref[ERTS_MAX_REF_NUMBERS];
    union {
	struct {
	    Eterm from;
	    ErlIOVec *evp;
	    ErlDrvBinary *cbinp;
	} outputv;
	struct {
	    Eterm from;
	    char *bufp;
	    ErlDrvSizeT size;
	} output;
	struct {
	    Eterm from;
	    Eterm connected;
	} connect;
	struct {
	    Eterm from;
	    Eterm reason;
	    ErlHeapFragment *bp;
	} exit;
	struct {
	    struct binary *binp;
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} control;
	struct {
	    unsigned int command;
	    char *bufp;
	    ErlDrvSizeT size;
	} call;
	struct {
	    Eterm item;
	} info;
	struct {
	    Eterm port;
	    Eterm to;
	} link;
	struct {
	    Eterm from;
	} unlink;
    } u;
} ;

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp);
ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proc2port_sig_is_command_op(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return !0;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return !0;
    default:				return 0;
    }
}

ERTS_GLB_INLINE ErlDrvSizeT
erts_proc2port_sig_command_data_size(ErtsProc2PortSigData *sigdp)
{
    switch (sigdp->flags & ERTS_P2P_SIG_TYPE_MASK) {
    case ERTS_P2P_SIG_TYPE_OUTPUT:	return sigdp->u.output.size;
    case ERTS_P2P_SIG_TYPE_OUTPUTV:	return sigdp->u.outputv.evp->size;
    default:				return (ErlDrvSizeT) 0;
    }
}

#endif
# 882 "beam/erl_port.h"

#define ERTS_PROC2PORT_SIG_EXEC			0
#define ERTS_PROC2PORT_SIG_ABORT		1
#define ERTS_PROC2PORT_SIG_ABORT_NOSUSPEND	2
#define ERTS_PROC2PORT_SIG_ABORT_CLOSED		3

typedef int (*ErtsProc2PortSigCallback)(Port *,
					erts_aint32_t,
					int,
					ErtsProc2PortSigData *);

typedef enum {
    ERTS_PORT_OP_BADARG,
    ERTS_PORT_OP_CALLER_EXIT,
    ERTS_PORT_OP_BUSY,
    ERTS_PORT_OP_BUSY_SCHEDULED,
    ERTS_PORT_OP_SCHEDULED,
    ERTS_PORT_OP_DROPPED,
    ERTS_PORT_OP_DONE
} ErtsPortOpResult;

ErtsPortOpResult
erts_schedule_proc2port_signal(Process *,
			       Port *,
			       Eterm,
			       Eterm *,
			       ErtsProc2PortSigData *,
			       int,
			       ErtsPortTaskHandle *,
			       ErtsProc2PortSigCallback);

int erts_deliver_port_exit(Port *, Eterm, Eterm, int);

/*
 * Port signal flags
 */
#define ERTS_PORT_SIG_FLG_BANG_OP		ERTS_P2P_SIG_DATA_FLG_BANG_OP
#define ERTS_PORT_SIG_FLG_NOSUSPEND		ERTS_P2P_SIG_DATA_FLG_NOSUSPEND
#define ERTS_PORT_SIG_FLG_FORCE			ERTS_P2P_SIG_DATA_FLG_FORCE
#define ERTS_PORT_SIG_FLG_BROKEN_LINK		ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK
#define ERTS_PORT_SIG_FLG_BAD_OUTPUT		ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT
#define ERTS_PORT_SIG_FLG_FORCE_SCHED		ERTS_P2P_SIG_DATA_FLG_SCHED
#define ERTS_PORT_SIG_FLG_ASYNC			ERTS_P2P_SIG_DATA_FLG_ASYNC
/* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */
#define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL	ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT

/*
 * Port ! {Owner, {command, Data}}
 * Port ! {Owner, {connect, NewOwner}}
 * Port ! {Owner, close}
 */
ErtsPortOpResult erts_port_command(Process *, int, Port *, Eterm, Eterm *);

/*
 * Signals from processes to ports.
 */
ErtsPortOpResult erts_port_output(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_exit(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_connect(Process *, int, Port *, Eterm, Eterm, Eterm *);
ErtsPortOpResult erts_port_link(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *);
ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *);
ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *);

#endif
# 948 "beam/erl_port.h"
# 45 "beam/global.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 45 "beam/global.h"
# 46 "beam/global.h"

struct enif_environment_t /* ErlNifEnv */
{
    struct erl_module_nif* mod_nif;
    Process* proc;
    Eterm* hp;
    Eterm* hp_end;
    ErlHeapFragment* heap_frag;
    int fpe_was_unmasked;
    struct enif_tmp_obj_t* tmp_obj_list;
    int exception_thrown; /* boolean */
};
extern void erts_pre_nif(struct enif_environment_t*, Process*,
			 struct erl_module_nif*);
extern void erts_post_nif(struct enif_environment_t* env);
extern Eterm erts_nif_taints(Process* p);
extern void erts_print_nif_taints(int to, void* to_arg);
void erts_unload_nif(struct erl_module_nif* nif);
extern void erl_nif_init(void);

/* Driver handle (wrapper for old plain handle) */
#define ERL_DE_OK      0
#define ERL_DE_UNLOAD  1
#define ERL_DE_FORCE_UNLOAD 2 
#define ERL_DE_RELOAD  3
#define ERL_DE_FORCE_RELOAD  4
#define ERL_DE_PERMANENT 5

#define ERL_DE_PROC_LOADED 0
#define ERL_DE_PROC_AWAIT_UNLOAD 1
#define ERL_DE_PROC_AWAIT_UNLOAD_ONLY 2
#define ERL_DE_PROC_AWAIT_LOAD 3

/* Flags for process entries */
#define ERL_DE_FL_DEREFERENCED 1

/* Flags for drivers, put locking policy here /PaN */
#define ERL_DE_FL_KILL_PORTS 1

#define ERL_FL_CONSISTENT_MASK ( ERL_DE_FL_KILL_PORTS )

/* System specific load errors are returned as positive values */
#define ERL_DE_NO_ERROR 0
#define ERL_DE_LOAD_ERROR_NO_INIT -1
#define ERL_DE_LOAD_ERROR_FAILED_INIT -2
#define ERL_DE_LOAD_ERROR_BAD_NAME -3
#define ERL_DE_LOAD_ERROR_NAME_TO_LONG -4
#define ERL_DE_LOAD_ERROR_INCORRECT_VERSION -5
#define ERL_DE_ERROR_NO_DDLL_FUNCTIONALITY -6
#define ERL_DE_ERROR_UNSPECIFIED -7
#define ERL_DE_LOOKUP_ERROR_NOT_FOUND -8
#define ERL_DE_DYNAMIC_ERROR_OFFSET -10

typedef struct de_proc_entry {
    Process *proc;                   /* The process... */
    Uint    awaiting_status;         /* PROC_LOADED == Have loaded the driver
			                PROC_AWAIT_UNLOAD == Wants to be notified 
			                when we have unloaded the driver (was locked)
			                PROC_AWAIT_LOAD == Wants to be notified when we
			                reloaded the driver (old was locked) */
    Uint    flags;                   /* ERL_FL_DE_DEREFERENCED when reload in progress */
    Eterm   heap[REF_THING_SIZE];    /* "ref heap" */
    struct  de_proc_entry *next;
} DE_ProcEntry;

typedef struct {
    void         *handle;             /* Handle for DLL or SO (for dyn. drivers). */
    DE_ProcEntry *procs;              /* List of pids that have loaded this driver,
				         or that wait for it to change state */
    erts_refc_t  refc;                /* Number of ports/processes having
					 references to the driver */
    erts_smp_atomic32_t port_count;   /* Number of ports using the driver */
    Uint         flags;               /* ERL_DE_FL_KILL_PORTS */
    int          status;              /* ERL_DE_xxx */
    char         *full_path;          /* Full path of the driver */
    char         *reload_full_path;   /* If status == ERL_DE_RELOAD, this contains
				         full name of driver (path) */
    char         *reload_driver_name; /* ... and this contains the driver name */
    Uint         reload_flags;        /* flags for reloaded driver */
} DE_Handle;

/*
 * This structure represents a link to the next driver.
 */

struct erts_driver_t_ {
    erts_driver_t *next;
    erts_driver_t *prev;
    char *name;
    struct {
	int major;
	int minor;
    } version;
    int flags;
    DE_Handle *handle;
#ifdef ERTS_SMP
    erts_smp_mtx_t *lock;
#endif
# 144 "beam/global.h"
    ErlDrvEntry *entry;
    ErlDrvData (*start)(ErlDrvPort port, char *command, SysDriverOpts* opts);
    void (*stop)(ErlDrvData drv_data);
    void (*finish)(void);
    void (*flush)(ErlDrvData drv_data);
    void (*output)(ErlDrvData drv_data, char *buf, ErlDrvSizeT len);
    void (*outputv)(ErlDrvData drv_data, ErlIOVec *ev); /* Might be NULL */
    ErlDrvSSizeT (*control)(ErlDrvData drv_data, unsigned int command,
			    char *buf, ErlDrvSizeT len,
			    char **rbuf, ErlDrvSizeT rlen); /* Might be NULL */
    ErlDrvSSizeT (*call)(ErlDrvData drv_data, unsigned int command,
			 char *buf, ErlDrvSizeT len,
			 char **rbuf, ErlDrvSizeT rlen, /* Might be NULL */
			 unsigned int *flags);
    void (*event)(ErlDrvData drv_data, ErlDrvEvent event,
		  ErlDrvEventData event_data);
    void (*ready_input)(ErlDrvData drv_data, ErlDrvEvent event); 
    void (*ready_output)(ErlDrvData drv_data, ErlDrvEvent event);  
    void (*timeout)(ErlDrvData drv_data);
    void (*ready_async)(ErlDrvData drv_data, ErlDrvThreadData thread_data); /* Might be NULL */ 
    void (*process_exit)(ErlDrvData drv_data, ErlDrvMonitor *monitor);
    void (*stop_select)(ErlDrvEvent event, void*); /* Might be NULL */
    void (*emergency_close)(ErlDrvData drv_data);  /* Might be NULL */
};

extern erts_driver_t *driver_list;
extern erts_smp_rwmtx_t erts_driver_list_lock;

extern void erts_ddll_init(void);
extern void erts_ddll_lock_driver(DE_Handle *dh, char *name);

/* These are for bookkeeping */
extern void erts_ddll_increment_port_count(DE_Handle *dh);
extern void erts_ddll_decrement_port_count(DE_Handle *dh);

/* These makes things happen, drivers may be scheduled for unload etc */
extern void erts_ddll_reference_driver(DE_Handle *dh);
extern void erts_ddll_reference_referenced_driver(DE_Handle *dh);
extern void erts_ddll_dereference_driver(DE_Handle *dh);

extern char *erts_ddll_error(int code);
extern void erts_ddll_proc_dead(Process *p, ErtsProcLocks plocks);
extern int erts_ddll_driver_ok(DE_Handle *dh);
extern void erts_ddll_remove_monitor(Process *p,
				     Eterm ref,
				     ErtsProcLocks plocks);
extern Eterm erts_ddll_monitor_driver(Process *p,
				      Eterm description,
				      ErtsProcLocks plocks);

/*
** Just like the driver binary but with initial flags
** Note that the two structures Binary and ErlDrvBinary HAVE to
** be equal except for extra fields in the beginning of the struct.
** ErlDrvBinary is defined in erl_driver.h.
** When driver_alloc_binary is called, a Binary is allocated, but 
** the pointer returned is to the address of the first element that
** also occurs in the ErlDrvBinary struct (driver.*binary takes care if this).
** The driver need never know about additions to the internal Binary of the
** emulator. One should however NEVER be sloppy when mixing ErlDrvBinary
** and Binary, the macros below can convert one type to the other, as they both
** in reality are equal.
*/

#ifdef ARCH_32
 /* *DO NOT USE* only for alignment. */
#define ERTS_BINARY_STRUCT_ALIGNMENT Uint32 align__;
#else
# 212 "beam/global.h"
#define ERTS_BINARY_STRUCT_ALIGNMENT
#endif
# 214 "beam/global.h"

/* Add fields in ERTS_INTERNAL_BINARY_FIELDS, otherwise the drivers crash */
#define ERTS_INTERNAL_BINARY_FIELDS				\
    UWord flags;							\
    erts_refc_t refc;						\
    ERTS_BINARY_STRUCT_ALIGNMENT

typedef struct binary {
    ERTS_INTERNAL_BINARY_FIELDS
    SWord orig_size;
    char orig_bytes[1]; /* to be continued */
} Binary;

#define ERTS_SIZEOF_Binary(Sz) \
    (offsetof(Binary,orig_bytes) + (Sz))

typedef struct {
    ERTS_INTERNAL_BINARY_FIELDS
    SWord orig_size;
    void (*destructor)(Binary *);
    union {
        struct {
            ERTS_BINARY_STRUCT_ALIGNMENT
            char data[1];
        } aligned;
        struct {
            char data[1];
        } unaligned;
    } u;
} ErtsMagicBinary;

#ifdef ARCH_32
#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 4
#else
# 248 "beam/global.h"
#define ERTS_MAGIC_BIN_BYTES_TO_ALIGN 0
#endif
# 250 "beam/global.h"

typedef union {
    Binary binary;
    ErtsMagicBinary magic_binary;
    struct {
	ERTS_INTERNAL_BINARY_FIELDS
	ErlDrvBinary binary;
    } driver;
} ErtsBinary;

/*
 * 'Binary' alignment:
 *   Address of orig_bytes[0] of a Binary should always be 8-byte aligned.
 * It is assumed that the flags, refc, and orig_size fields are 4 bytes on
 * 32-bits architectures and 8 bytes on 64-bits architectures.
 */

#define ERTS_MAGIC_BIN_DESTRUCTOR(BP) \
  ((ErtsBinary *) (BP))->magic_binary.destructor
#define ERTS_MAGIC_BIN_DATA(BP) \
  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.aligned.data)
#define ERTS_MAGIC_DATA_OFFSET \
  (offsetof(ErtsMagicBinary,u.aligned.data) - offsetof(Binary,orig_bytes))
#define ERTS_MAGIC_BIN_ORIG_SIZE(Sz) \
  (ERTS_MAGIC_DATA_OFFSET + (Sz))
#define ERTS_MAGIC_BIN_SIZE(Sz) \
  (offsetof(ErtsMagicBinary,u.aligned.data) + (Sz))

/* On 32-bit arch these macro variants will save memory
   by not forcing 8-byte alignment for the magic payload.
*/
#define ERTS_MAGIC_BIN_UNALIGNED_DATA(BP) \
  ((void *) ((ErtsBinary *) (BP))->magic_binary.u.unaligned.data)
#define ERTS_MAGIC_UNALIGNED_DATA_OFFSET \
  (offsetof(ErtsMagicBinary,u.unaligned.data) - offsetof(Binary,orig_bytes))
#define ERTS_MAGIC_BIN_UNALIGNED_DATA_SIZE(BP) \
  ((BP)->orig_size - ERTS_MAGIC_UNALIGNED_DATA_OFFSET)
#define ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(Sz) \
  (ERTS_MAGIC_UNALIGNED_DATA_OFFSET + (Sz))
#define ERTS_MAGIC_BIN_UNALIGNED_SIZE(Sz) \
  (offsetof(ErtsMagicBinary,u.unaligned.data) + (Sz))
#define ERTS_MAGIC_BIN_FROM_UNALIGNED_DATA(DATA) \
  ((ErtsBinary*)((char*)(DATA) - offsetof(ErtsMagicBinary,u.unaligned.data)))


#define Binary2ErlDrvBinary(B) (&((ErtsBinary *) (B))->driver.binary)
#define ErlDrvBinary2Binary(D) ((Binary *) \
				(((char *) (D)) \
				 - offsetof(ErtsBinary, driver.binary)))

/* A "magic" binary flag */
#define BIN_FLAG_MAGIC      1
#define BIN_FLAG_USR1       2 /* Reserved for use by different modules too mark */
#define BIN_FLAG_USR2       4 /*  certain binaries as special (used by ets) */
#define BIN_FLAG_DRV        8

/*
 * This structure represents one type of a binary in a process.
 */

typedef struct proc_bin {
    Eterm thing_word;		/* Subtag REFC_BINARY_SUBTAG. */
    Uint size;			/* Binary size in bytes. */
#if HALFWORD_HEAP
    void* dummy_ptr_padding__;
#endif
# 316 "beam/global.h"
    struct erl_off_heap_header *next;
    Binary *val;		/* Pointer to Binary structure. */
    byte *bytes;		/* Pointer to the actual data bytes. */
    Uint flags;			/* Flag word. */
} ProcBin;

#define PB_IS_WRITABLE 1	/* Writable (only one reference to ProcBin) */
#define PB_ACTIVE_WRITER 2	/* There is an active writer */

/*
 * ProcBin size in Eterm words.
 */
#define PROC_BIN_SIZE (sizeof(ProcBin)/sizeof(Eterm))

ERTS_GLB_INLINE Eterm erts_mk_magic_binary_term(Eterm **hpp,
						ErlOffHeap *ohp,
						Binary *mbp);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Eterm
erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp)
{
    ProcBin *pb = (ProcBin *) *hpp;
    *hpp += PROC_BIN_SIZE;

    ASSERT(mbp->flags & BIN_FLAG_MAGIC);

    pb->thing_word = HEADER_PROC_BIN;
    pb->size = 0;
    pb->next = ohp->first;
    ohp->first = (struct erl_off_heap_header*) pb;
    pb->val = mbp;
    pb->bytes = (byte *) mbp->orig_bytes;
    pb->flags = 0;

    erts_refc_inc(&mbp->refc, 1);

    return make_binary(pb);    
}

#endif
# 358 "beam/global.h"

#define ERTS_TERM_IS_MAGIC_BINARY(T) \
  (is_binary((T)) \
   && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \
   && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC))


union erl_off_heap_ptr {
    struct erl_off_heap_header* hdr;
    ProcBin *pb;
    struct erl_fun_thing* fun;
    struct external_thing_* ext;
    Eterm* ep;
    void* voidp;
};

/* controls warning mapping in error_logger */

extern Eterm node_cookie;
extern Uint display_items;	/* no of items to display in traces etc */

extern int erts_backtrace_depth;
extern erts_smp_atomic32_t erts_max_gen_gcs;

extern int bif_reductions;      /* reductions + fcalls (when doing call_bif) */
extern int stackdump_on_exit;

/*
 * Here is an implementation of a lightweiht stack.
 *
 * Use it like this:
 *
 * DECLARE_ESTACK(Stack)	(At the start of a block)
 * ...
 * ESTACK_PUSH(Stack, Term)
 * ...
 * if (ESTACK_ISEMPTY(Stack)) {
 *    Stack is empty
 * } else {
 *    Term = ESTACK_POP(Stack);
 *    Process popped Term here
 * }
 * ...
 * DESTROY_ESTACK(Stack)
 */

typedef struct ErtsEStack_ {
    Eterm* start;
    Eterm* sp;
    Eterm* end;
    Eterm* edefault;
    ErtsAlcType_t alloc_type;
}ErtsEStack;

#define DEF_ESTACK_SIZE (16)

void erl_grow_estack(ErtsEStack*, Uint need);
#define ESTK_CONCAT(a,b) a##b
#define ESTK_DEF_STACK(s) ESTK_CONCAT(s,_default_estack)

#define DECLARE_ESTACK(s)				\
    Eterm ESTK_DEF_STACK(s)[DEF_ESTACK_SIZE];		\
    ErtsEStack s = {					\
        ESTK_DEF_STACK(s),  /* start */ 		\
        ESTK_DEF_STACK(s),  /* sp */			\
        ESTK_DEF_STACK(s) + DEF_ESTACK_SIZE, /* end */	\
        ESTK_DEF_STACK(s),  /* default */		\
        ERTS_ALC_T_ESTACK /* alloc_type */		\
    }

#define ESTACK_CHANGE_ALLOCATOR(s,t)					\
do {									\
    if ((s).start != ESTK_DEF_STACK(s)) {				\
	erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator "	\
		 "type of active estack\n");				\
    }									\
    (s).alloc_type = (t);						\
 } while (0)

#define DESTROY_ESTACK(s)				\
do {							\
    if ((s).start != ESTK_DEF_STACK(s)) {		\
	erts_free((s).alloc_type, (s).start); 		\
    }							\
} while(0)


/*
 * Do not free the stack after this, it may have pointers into what
 * was saved in 'dst'.
 */
#define ESTACK_SAVE(s,dst)\
do {\
    if ((s).start == ESTK_DEF_STACK(s)) {\
	UWord _wsz = ESTACK_COUNT(s);\
	(dst)->start = erts_alloc((s).alloc_type,\
				  DEF_ESTACK_SIZE * sizeof(Eterm));\
	memcpy((dst)->start, (s).start,_wsz*sizeof(Eterm));\
	(dst)->sp = (dst)->start + _wsz;\
	(dst)->end = (dst)->start + DEF_ESTACK_SIZE;\
        (dst)->edefault = NULL;\
	(dst)->alloc_type = (s).alloc_type;\
    } else\
        *(dst) = (s);\
 } while (0)

#define DESTROY_SAVED_ESTACK(estack)\
do {\
    if ((estack)->start) {\
	erts_free((estack)->alloc_type, (estack)->start);\
	(estack)->start = NULL;\
    }\
} while(0)

#define CLEAR_SAVED_ESTACK(estack) ((void) ((estack)->start = NULL))

/*
 * Use on empty stack, only the allocator can be changed before this.
 * The src stack is reset to NULL.
 */
#define ESTACK_RESTORE(s, src)			\
do {						\
    ASSERT((s).start == ESTK_DEF_STACK(s));	\
    (s) = *(src);  /* struct copy */		\
    (src)->start = NULL;			\
    ASSERT((s).sp >= (s).start);		\
    ASSERT((s).sp <= (s).end);			\
} while (0)

#define ESTACK_IS_STATIC(s) ((s).start == ESTK_DEF_STACK(s))

#define ESTACK_PUSH(s, x)			\
do {						\
    if ((s).sp == (s).end) {			\
	erl_grow_estack(&(s), 1); 		\
    }						\
    *(s).sp++ = (x);				\
} while(0)

#define ESTACK_PUSH2(s, x, y)			\
do {						\
    if ((s).sp > (s).end - 2) {			\
	erl_grow_estack(&(s), 2);		\
    }						\
    *(s).sp++ = (x);				\
    *(s).sp++ = (y);				\
} while(0)

#define ESTACK_PUSH3(s, x, y, z)		\
do {						\
    if ((s).sp > (s).end - 3) {			\
	erl_grow_estack(&s, 3); 		\
    }						\
    *(s).sp++ = (x);				\
    *(s).sp++ = (y);				\
    *(s).sp++ = (z);				\
} while(0)

#define ESTACK_PUSH4(s, E1, E2, E3, E4)		\
do {						\
    if ((s).sp > (s).end - 4) {			\
	erl_grow_estack(&s, 4);                 \
    }						\
    *(s).sp++ = (E1);				\
    *(s).sp++ = (E2);				\
    *(s).sp++ = (E3);				\
    *(s).sp++ = (E4);				\
} while(0)

#define ESTACK_RESERVE(s, push_cnt)             \
do {					        \
    if ((s).sp > (s).end - (push_cnt)) {	\
	erl_grow_estack(&(s), (push_cnt));	\
    }					        \
} while(0)

/* Must be preceded by ESTACK_RESERVE */
#define ESTACK_FAST_PUSH(s, x)				\
do {							\
    ASSERT((s).sp < (s).end);                           \
    *s.sp++ = (x);					\
} while(0)

#define ESTACK_COUNT(s) ((s).sp - (s).start)
#define ESTACK_ISEMPTY(s) ((s).sp == (s).start)
#define ESTACK_POP(s) (*(--(s).sp))


/*
 * WSTACK: same as ESTACK but with UWord instead of Eterm
 */

typedef struct ErtsWStack_ {
    UWord* wstart;
    UWord* wsp;
    UWord* wend;
    UWord* wdefault;
    ErtsAlcType_t alloc_type;
}ErtsWStack;

#define DEF_WSTACK_SIZE (16)

void erl_grow_wstack(ErtsWStack*, Uint need);
#define WSTK_CONCAT(a,b) a##b
#define WSTK_DEF_STACK(s) WSTK_CONCAT(s,_default_wstack)

#define WSTACK_DECLARE(s)				\
    UWord WSTK_DEF_STACK(s)[DEF_WSTACK_SIZE];		\
    ErtsWStack s = {					\
        WSTK_DEF_STACK(s),  /* wstart */ 		\
        WSTK_DEF_STACK(s),  /* wsp */			\
        WSTK_DEF_STACK(s) + DEF_WSTACK_SIZE, /* wend */	\
        WSTK_DEF_STACK(s),  /* wdflt */ 		\
        ERTS_ALC_T_ESTACK /* alloc_type */		\
    }
#define DECLARE_WSTACK WSTACK_DECLARE

typedef struct ErtsDynamicWStack_ {
    UWord default_stack[DEF_WSTACK_SIZE];
    ErtsWStack ws;
}ErtsDynamicWStack;

#define WSTACK_INIT(dwsp, ALC_TYPE)                               \
do {	 	                                                  \
    (dwsp)->ws.wstart   = (dwsp)->default_stack;                  \
    (dwsp)->ws.wsp      = (dwsp)->default_stack;                  \
    (dwsp)->ws.wend     = (dwsp)->default_stack + DEF_WSTACK_SIZE;\
    (dwsp)->ws.wdefault = (dwsp)->default_stack;                  \
    (dwsp)->ws.alloc_type = ALC_TYPE;                             \
} while (0)

#define WSTACK_CHANGE_ALLOCATOR(s,t)					\
do {									\
    if (s.wstart != WSTK_DEF_STACK(s)) {				\
	erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator "	\
		 "type of active wstack\n");				\
    }									\
    s.alloc_type = (t);							\
 } while (0)

#define WSTACK_DESTROY(s)				\
do {							\
    if (s.wstart != s.wdefault) {		        \
	erts_free(s.alloc_type, s.wstart); 		\
    }							\
} while(0)
#define DESTROY_WSTACK WSTACK_DESTROY

#define WSTACK_DEBUG(s) \
    do { \
	fprintf(stderr, "wstack size   = %ld\r\n", s.wsp - s.wstart); \
	fprintf(stderr, "wstack wstart = %p\r\n", s.wstart); \
	fprintf(stderr, "wstack wsp    = %p\r\n", s.wsp); \
    } while(0)

/*
 * Do not free the stack after this, it may have pointers into what
 * was saved in 'dst'.
 */
#define WSTACK_SAVE(s,dst)\
do {\
    if (s.wstart == WSTK_DEF_STACK(s)) {\
	UWord _wsz = WSTACK_COUNT(s);\
	(dst)->wstart = erts_alloc(s.alloc_type,\
				  DEF_WSTACK_SIZE * sizeof(UWord));\
	memcpy((dst)->wstart, s.wstart,_wsz*sizeof(UWord));\
	(dst)->wsp = (dst)->wstart + _wsz;\
	(dst)->wend = (dst)->wstart + DEF_WSTACK_SIZE;\
        (dst)->wdefault = NULL;\
	(dst)->alloc_type = s.alloc_type;\
    } else\
        *(dst) = s;\
 } while (0)

#define DESTROY_SAVED_WSTACK(wstack)\
do {\
    if ((wstack)->wstart) {\
	erts_free((wstack)->alloc_type, (wstack)->wstart);\
	(wstack)->wstart = NULL;\
    }\
} while(0)

#define CLEAR_SAVED_WSTACK(wstack) ((void) ((wstack)->wstart = NULL))

/*
 * Use on empty stack, only the allocator can be changed before this.
 * The src stack is reset to NULL.
 */
#define WSTACK_RESTORE(s, src)			\
do {						\
    ASSERT(s.wstart == WSTK_DEF_STACK(s));	\
    s = *(src);  /* struct copy */		\
    (src)->wstart = NULL;			\
    ASSERT(s.wsp >= s.wstart);			\
    ASSERT(s.wsp <= s.wend);			\
} while (0)

#define WSTACK_IS_STATIC(s) (s.wstart == WSTK_DEF_STACK(s))

#define WSTACK_PUSH(s, x)				\
do {							\
    if (s.wsp == s.wend) {				\
	erl_grow_wstack(&s, 1); 	                \
    }							\
    *s.wsp++ = (x);					\
} while(0)

#define WSTACK_PUSH2(s, x, y)			\
do {						\
    if (s.wsp > s.wend - 2) {			\
	erl_grow_wstack(&s, 2);                 \
    }						\
    *s.wsp++ = (x);				\
    *s.wsp++ = (y);				\
} while(0)

#define WSTACK_PUSH3(s, x, y, z)		\
do {						\
    if (s.wsp > s.wend - 3) {	                \
	erl_grow_wstack(&s, 3);                 \
    }						\
    *s.wsp++ = (x);				\
    *s.wsp++ = (y);				\
    *s.wsp++ = (z);				\
} while(0)

#define WSTACK_PUSH4(s, A1, A2, A3, A4)		\
do {						\
    if (s.wsp > s.wend - 4) {	                \
	erl_grow_wstack(&s, 4);                 \
    }						\
    *s.wsp++ = (A1);				\
    *s.wsp++ = (A2);				\
    *s.wsp++ = (A3);				\
    *s.wsp++ = (A4);				\
} while(0)

#define WSTACK_PUSH5(s, A1, A2, A3, A4, A5)     \
do {						\
    if (s.wsp > s.wend - 5) {	                \
	erl_grow_wstack(&s, 5);                 \
    }						\
    *s.wsp++ = (A1);				\
    *s.wsp++ = (A2);				\
    *s.wsp++ = (A3);				\
    *s.wsp++ = (A4);				\
    *s.wsp++ = (A5);				\
} while(0)

#define WSTACK_PUSH6(s, A1, A2, A3, A4, A5, A6) \
do {						\
    if (s.wsp > s.wend - 6) {	                \
	erl_grow_wstack(&s, 6);                 \
    }						\
    *s.wsp++ = (A1);				\
    *s.wsp++ = (A2);				\
    *s.wsp++ = (A3);				\
    *s.wsp++ = (A4);				\
    *s.wsp++ = (A5);				\
    *s.wsp++ = (A6);				\
} while(0)

#define WSTACK_RESERVE(s, push_cnt)             \
do {						\
    if (s.wsp > s.wend - (push_cnt)) { 	        \
	erl_grow_wstack(&s, (push_cnt));        \
    }                                           \
} while(0)

/* Must be preceded by WSTACK_RESERVE */
#define WSTACK_FAST_PUSH(s, x)                  \
do {						\
    ASSERT(s.wsp < s.wend);                     \
    *s.wsp++ = (x);                             \
} while(0)

#define WSTACK_COUNT(s) (s.wsp - s.wstart)
#define WSTACK_ISEMPTY(s) (s.wsp == s.wstart)
#define WSTACK_POP(s) ((ASSERT(s.wsp > s.wstart)),*(--s.wsp))

#define WSTACK_ROLLBACK(s, count) (ASSERT(WSTACK_COUNT(s) >= (count)), \
                                   s.wsp = s.wstart + (count))

/* PSTACK - Stack of any type.
 * Usage:
 * {
 * #define PSTACK_TYPE MyType
 *    PSTACK_DECLARE(s,16);
 *    MyType *sp = PSTACK_PUSH(s);
 *
 *    sp->x = ....
 *    sp->y = ....
 *    sp = PSTACK_PUSH(s);
 *    ...
 *    sp = PSTACK_POP(s);
 *    if (PSTACK_IS_EMPTY(s)) {
 *        // sp is invalid when stack is empty after pop
 *    }
 *
 *    PSTACK_DESTROY(s);
 * }
 */


typedef struct ErtsPStack_ {
    byte* pstart;
    byte* psp;
    byte* pend;
    ErtsAlcType_t alloc_type;
}ErtsPStack;

void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes);
#define PSTK_CONCAT(a,b) a##b
#define PSTK_DEF_STACK(s) PSTK_CONCAT(s,_default_pstack)

#define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \
PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE];                            \
ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */                    \
                 (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */                 \
                 (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\
                 ERTS_ALC_T_ESTACK   /* alloc_type */                      \
}

#define PSTACK_CHANGE_ALLOCATOR(s,t)					\
do {									\
    if (s.pstart != (byte*)PSTK_DEF_STACK(s)) {				\
	erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator "	\
		 "type of active pstack\n");				\
    }									\
    s.alloc_type = (t);							\
 } while (0)

#define PSTACK_DESTROY(s)				\
do {							\
    if (s.pstart != (byte*)PSTK_DEF_STACK(s)) {		\
	erts_free(s.alloc_type, s.pstart); 		\
    }							\
} while(0)

#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart)

#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart)

#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp))

#define PSTACK_PUSH(s) 		                                           \
    (s.psp += sizeof(PSTACK_TYPE),                                         \
     ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s),           \
                                          sizeof(PSTACK_TYPE)) : (void)0), \
     ((PSTACK_TYPE*) s.psp))

#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE)))

/*
 * Do not free the stack after this, it may have pointers into what
 * was saved in 'dst'.
 */
#define PSTACK_SAVE(s,dst)\
do {\
    if (s.pstart == (byte*)PSTK_DEF_STACK(s)) {\
	UWord _pbytes = PSTACK_COUNT(s) * sizeof(PSTACK_TYPE);\
	(dst)->pstart = erts_alloc(s.alloc_type,\
				   sizeof(PSTK_DEF_STACK(s)));\
	sys_memcpy((dst)->pstart, s.pstart, _pbytes);\
	(dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\
	(dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\
	(dst)->alloc_type = s.alloc_type;\
    } else\
        *(dst) = s;\
 } while (0)

/*
 * Use on empty stack, only the allocator can be changed before this.
 * The src stack is reset to NULL.
 */
#define PSTACK_RESTORE(s, src)			        \
do {						        \
    ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s));	\
    s = *(src);  /* struct copy */		        \
    (src)->pstart = NULL;			        \
    ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE)));  \
    ASSERT(s.psp < s.pend);			        \
} while (0)

#define PSTACK_DESTROY_SAVED(pstack)\
do {\
    if ((pstack)->pstart) {\
	erts_free((pstack)->alloc_type, (pstack)->pstart);\
	(pstack)->pstart = NULL;\
    }\
} while(0)


/* binary.c */

void erts_emasculate_writable_binary(ProcBin* pb);
Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap);
Eterm erts_new_mso_binary(Process*, byte*, int);
Eterm new_binary(Process*, byte*, Uint);
Eterm erts_realloc_binary(Eterm bin, size_t size);

/* erl_bif_info.c */

Eterm
erts_bld_port_info(Eterm **hpp,
		   ErlOffHeap *ohp,
		   Uint *szp,
		   Port *prt,
		   Eterm item); 

void erts_bif_info_init(void);

/* bif.c */

ERTS_GLB_INLINE Eterm
erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS]);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE Eterm
erts_proc_store_ref(Process *c_p, Uint32 ref[ERTS_MAX_REF_NUMBERS])
{
    Eterm *hp = HAlloc(c_p, REF_THING_SIZE);
    write_ref_thing(hp, ref[0], ref[1], ref[2]);
    return make_internal_ref(hp);
}

#endif
# 886 "beam/global.h"

void erts_queue_monitor_message(Process *,
				ErtsProcLocks*,
				Eterm,
				Eterm,
				Eterm,
				Eterm);
void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
			   Eterm (*bif)(Process*,Eterm*));
void erts_init_bif(void);
Eterm erl_send(Process *p, Eterm to, Eterm msg);

/* erl_bif_op.c */

Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2);

/* beam_bif_load.c */
Eterm erts_check_process_code(Process *c_p, Eterm module, int allow_gc, int *redsp);


/* beam_load.c */
typedef struct {
    BeamInstr* current;		/* Pointer to: Mod, Name, Arity */
    Uint needed;		/* Heap space needed for entire tuple */
    Uint32 loc;			/* Location in source code */
    Eterm* fname_ptr;		/* Pointer to fname table */
} FunctionInfo;

Binary* erts_alloc_loader_state(void);
Eterm erts_module_for_prepared_code(Binary* magic);
Eterm erts_prepare_loading(Binary* loader_state,  Process *c_p,
			   Eterm group_leader, Eterm* modp,
			   byte* code, Uint size);
Eterm erts_finish_loading(Binary* loader_state, Process* c_p,
			  ErtsProcLocks c_p_locks, Eterm* modp);
Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks,
			  Eterm group_leader, Eterm* mod, byte* code, Uint size);
void init_load(void);
BeamInstr* find_function_from_pc(BeamInstr* pc);
Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp,
			   Eterm args, Eterm* mfa_p);
void erts_set_current_function(FunctionInfo* fi, BeamInstr* current);
Eterm erts_module_info_0(Process* p, Eterm module);
Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);

/* beam_ranges.c */
void erts_init_ranges(void);
void erts_start_staging_ranges(void);
void erts_end_staging_ranges(int commit);
void erts_update_ranges(BeamInstr* code, Uint size);
void erts_remove_from_ranges(BeamInstr* code);
UWord erts_ranges_sz(void);
void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info);

/* break.c */
void init_break_handler(void);
void erts_set_ignore_break(void);
void erts_replace_intr(void);
void process_info(int, void *);
void print_process_info(int, void *, Process*);
void info(int, void *);
void loaded(int, void *);

/* erl_arith.c */
double erts_get_positive_zero_float(void);

/* config.c */

__decl_noreturn void __noreturn erts_exit(int n, char*, ...);
__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...);
void erl_error(char*, va_list);

/* copy.c */
Eterm copy_object(Eterm, Process*);

#if HALFWORD_HEAP
Uint size_object_rel(Eterm, Eterm*);
#  define size_object(A) size_object_rel(A,NULL)

Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base);
#  define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL)

Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base);
#  define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL)

#else /* !HALFWORD_HEAP */
# 973 "beam/global.h"

Uint size_object(Eterm);
#  define size_object_rel(A,B) size_object(A)

Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*);
#  define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH)

Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*);
#  define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D)

#endif
# 984 "beam/global.h"


void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first,
		      Eterm* refs, unsigned nrefs);

/* Utilities */
extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks);
extern Eterm erts_monitor_nodes(Process *, Eterm, Eterm);
extern Eterm erts_processes_monitoring_nodes(Process *);
extern int erts_do_net_exits(DistEntry*, Eterm);
extern int distribution_info(int, void *);
extern int is_node_name_atom(Eterm a);

extern int erts_net_message(Port *, DistEntry *,
			    byte *, ErlDrvSizeT, byte *, ErlDrvSizeT);

extern void init_dist(void);
extern int stop_dist(void);

void erl_progressf(char* format, ...);

#ifdef MESS_DEBUG
void print_pass_through(int, byte*, int);
#endif
# 1008 "beam/global.h"

/* beam_emu.c */
int catchlevel(Process*);
void init_emulator(void);
void process_main(void);
Eterm build_stacktrace(Process* c_p, Eterm exc);
Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value);
void erts_save_stacktrace(Process* p, struct StackTrace* s, int depth);

/* erl_init.c */

typedef struct {
    Eterm delay_time;
    int context_reds;
    int input_reds;
} ErtsModifiedTimings;

extern Export *erts_delay_trap;
extern int erts_modified_timing_level;
extern ErtsModifiedTimings erts_modified_timings[];
#define ERTS_USE_MODIFIED_TIMING() \
  (erts_modified_timing_level >= 0)
#define ERTS_MODIFIED_TIMING_DELAY \
  (erts_modified_timings[erts_modified_timing_level].delay_time)
#define ERTS_MODIFIED_TIMING_CONTEXT_REDS \
  (erts_modified_timings[erts_modified_timing_level].context_reds)
#define ERTS_MODIFIED_TIMING_INPUT_REDS \
  (erts_modified_timings[erts_modified_timing_level].input_reds)

extern int erts_no_line_info;
extern Eterm erts_error_logger_warnings;
extern int erts_initialized;
extern int erts_compat_rel;
extern int erts_use_sender_punish;
void erts_short_init(void);
void erl_start(int, char**);
void erts_usage(void);
Eterm erts_preloaded(Process* p);
/* erl_md5.c */

typedef struct {
    Uint32 state[4];		/* state (ABCD) */
    Uint32 count[2];		/* number of bits, modulo 2^64 (lsb first) */
    unsigned char buffer[64];	/* input buffer */
} MD5_CTX;

void MD5Init(MD5_CTX *);
void MD5Update(MD5_CTX *, unsigned char *, unsigned int);
void MD5Final(unsigned char [16], MD5_CTX *);


/* io.c */

typedef struct {
    char *name;
    char *driver_name;
} ErtsPortNames;

#define ERTS_SPAWN_DRIVER 1
#define ERTS_SPAWN_EXECUTABLE 2
#define ERTS_SPAWN_ANY (ERTS_SPAWN_DRIVER | ERTS_SPAWN_EXECUTABLE)
int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_locked);
void erts_destroy_driver(erts_driver_t *drv);
int erts_save_suspend_process_on_port(Port*, Process*);
Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *);
void erts_init_io(int, int, int);
void erts_raw_port_command(Port*, byte*, Uint);
void driver_report_exit(ErlDrvPort, int);
LineBuf* allocate_linebuf(int);
int async_ready(Port *, void*);
ErtsPortNames *erts_get_port_names(Eterm, ErlDrvPort);
void erts_free_port_names(ErtsPortNames *);
Uint erts_port_ioq_size(Port *pp);
void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int);

Port *erts_get_heart_port(void);
void erts_emergency_close_ports(void);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_COUNT)
void erts_lcnt_enable_io_lock_count(int enable);
#endif
# 1089 "beam/global.h"

/* driver_tab.c */
typedef void *(*ErtsStaticNifInitFPtr)(void);
ErtsStaticNifInitFPtr erts_static_nif_get_nif_init(const char *name, int len);
int erts_is_static_nif(void *handle);
void erts_init_static_drivers(void);

/* erl_drv_thread.c */
void erl_drv_thr_init(void);

/* utils.c */
void erts_cleanup_offheap(ErlOffHeap *offheap);

Uint64 erts_timestamp_millis(void);

Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex);

Eterm store_external_or_ref_in_proc_(Process *, Eterm);
Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm);

#define NC_HEAP_SIZE(NC) \
 (ASSERT(is_node_container((NC))), \
  IS_CONST((NC)) ? 0 : (thing_arityval(*boxed_val((NC))) + 1))
#define STORE_NC(Hpp, ETpp, NC) \
 (ASSERT(is_node_container((NC))), \
  IS_CONST((NC)) ? (NC) : store_external_or_ref_((Hpp), (ETpp), (NC)))
#define STORE_NC_IN_PROC(Pp, NC) \
 (ASSERT(is_node_container((NC))), \
  IS_CONST((NC)) ? (NC) : store_external_or_ref_in_proc_((Pp), (NC)))

/* duplicates from big.h */
int term_to_Uint(Eterm term, Uint *up);
int term_to_UWord(Eterm, UWord*);

#ifdef HAVE_ERTS_NOW_CPU
extern int erts_cpu_timestamp;
#endif
# 1126 "beam/global.h"
/* erl_bif_chksum.c */
void erts_init_bif_chksum(void);
/* erl_bif_re.c */
void erts_init_bif_re(void);
Sint erts_re_set_loop_limit(Sint limit);
/* erl_bif_binary.c */
void erts_init_bif_binary(void);
Sint erts_binary_set_loop_limit(Sint limit);

/* external.c */
void erts_init_external(void);

/* erl_map.c */
void erts_init_map(void);

/* erl_unicode.c */
void erts_init_unicode(void);
Sint erts_unicode_set_loop_limit(Sint limit);

void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ;
Sint erts_native_filename_need(Eterm ioterm, int encoding);
void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars);
int erts_analyze_utf8(byte *source, Uint size, 
			byte **err_pos, Uint *num_chars, int *left);
int erts_analyze_utf8_x(byte *source, Uint size, 
			byte **err_pos, Uint *num_chars, int *left,
			Sint *num_latin1_chars, Uint max_chars);
char *erts_convert_filename_to_native(Eterm name, char *statbuf, 
				      size_t statbuf_size, 
				      ErtsAlcType_t alloc_type, 
				      int allow_empty, int allow_atom,
				      Sint *used /* out */);
char *erts_convert_filename_to_encoding(Eterm name, char *statbuf,
					size_t statbuf_size,
					ErtsAlcType_t alloc_type,
					int allow_empty, int allow_atom,
					int encoding,
					Sint *used /* out */,
					Uint extra);
char* erts_convert_filename_to_wchar(byte* bytes, Uint size,
                                     char *statbuf, size_t statbuf_size,
                                     ErtsAlcType_t alloc_type, Sint* used,
                                     Uint extra_wchars);
Eterm erts_convert_native_to_filename(Process *p, byte *bytes);
Eterm erts_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, Uint left,
			Uint *num_built, Uint *num_eaten, Eterm tail);
int erts_utf8_to_latin1(byte* dest, const byte* source, int slen);
#define ERTS_UTF8_OK 0
#define ERTS_UTF8_INCOMPLETE 1
#define ERTS_UTF8_ERROR 2
#define ERTS_UTF8_ANALYZE_MORE 3
#define ERTS_UTF8_OK_MAX_CHARS 4

void bin_write(int, void*, byte*, size_t);
int intlist_to_buf(Eterm, char*, int); /* most callers pass plain char*'s */

struct Sint_buf {
#if defined(ARCH_64) && !HALFWORD_HEAP
    char s[22];
#else
# 1186 "beam/global.h"
    char s[12];
#endif
# 1188 "beam/global.h"
};	
char* Sint_to_buf(Sint, struct Sint_buf*);

#define ERTS_IOLIST_STATE_INITER(C_P, OBJ)	\
    {(C_P), 0, 0, (OBJ), {NULL, NULL, NULL, ERTS_ALC_T_INVALID}, 0, 0}

#define ERTS_IOLIST_STATE_MOVE(TO, FROM)	\
    sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOListState))

#define ERTS_IOLIST_SIZE_YIELDS_COUNT_PER_RED 8

typedef struct {
    Process *c_p;
    ErlDrvSizeT size;
    Uint offs;
    Eterm obj;
    ErtsEStack estack;
    int reds_left;
    int have_size;
} ErtsIOListState;

#define ERTS_IOLIST2BUF_STATE_INITER(C_P, OBJ)	\
    {ERTS_IOLIST_STATE_INITER((C_P), (OBJ)), {NULL, 0, 0, 0}, NULL, 0, NULL, 0}

#define ERTS_IOLIST2BUF_STATE_MOVE(TO, FROM)	\
    sys_memcpy((void *) (TO), (void *) (FROM), sizeof(ErtsIOList2BufState))

#define ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT 32
#define ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED 8
#define ERTS_IOLIST_TO_BUF_BYTES_PER_RED \
    (ERTS_IOLIST_TO_BUF_YIELD_COUNT_PER_RED*ERTS_IOLIST_TO_BUF_BYTES_PER_YIELD_COUNT)

typedef struct {
    ErtsIOListState iolist;
    struct {
	byte *bptr;
	size_t size;
	Uint bitoffs;
	Uint bitsize;
    } bcopy;
    char *buf;
    ErlDrvSizeT len;
    Eterm *objp;
    int offset;
} ErtsIOList2BufState;

#define ERTS_IOLIST_OK 0
#define ERTS_IOLIST_OVERFLOW 1
#define ERTS_IOLIST_TYPE 2
#define ERTS_IOLIST_YIELD 3

Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */

#define ERTS_IOLIST_TO_BUF_OVERFLOW	(~((ErlDrvSizeT) 0))
#define ERTS_IOLIST_TO_BUF_TYPE_ERROR	(~((ErlDrvSizeT) 1))
#define ERTS_IOLIST_TO_BUF_YIELD	(~((ErlDrvSizeT) 2))
#define ERTS_IOLIST_TO_BUF_FAILED(R) \
    (((R) & (~((ErlDrvSizeT) 3))) == (~((ErlDrvSizeT) 3)))
#define ERTS_IOLIST_TO_BUF_SUCCEEDED(R) \
    (!ERTS_IOLIST_TO_BUF_FAILED((R)))

ErlDrvSizeT erts_iolist_to_buf(Eterm, char*, ErlDrvSizeT);
ErlDrvSizeT erts_iolist_to_buf_yielding(ErtsIOList2BufState *);
int erts_iolist_size_yielding(ErtsIOListState *state);
int erts_iolist_size(Eterm, ErlDrvSizeT *);
int is_string(Eterm);
void erl_at_exit(void (*) (void*), void*);
Eterm collect_memory(Process *);
void dump_memory_to_fd(int);
int dump_memory_data(const char *);

Eterm erts_mixed_plus(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_mixed_minus(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_mixed_times(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_mixed_div(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_int_div(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_int_rem(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_band(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_bor(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_bxor(Process* p, Eterm arg1, Eterm arg2);
Eterm erts_bnot(Process* p, Eterm arg);

Eterm erts_gc_mixed_plus(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_mixed_minus(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_mixed_times(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_mixed_div(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_int_div(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_int_rem(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_band(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_bor(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_bxor(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live);

Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live);
Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live);

Uint erts_current_reductions(Process* current, Process *p);

int erts_print_system_version(int to, void *arg, Process *c_p);

int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg);

/*
** Call_trace uses this API for the parameter matching functions
*/

#define MatchSetRef(MPSP) 			\
do {						\
    if ((MPSP) != NULL) {			\
	erts_refc_inc(&(MPSP)->refc, 1);	\
    }						\
} while (0)

#define MatchSetUnref(MPSP)					\
do {								\
    if (((MPSP) != NULL) && erts_refc_dectest(&(MPSP)->refc, 0) <= 0) { \
	erts_bin_free(MPSP);					\
    }								\
} while(0)

#define MatchSetGetSource(MPSP) erts_match_set_get_source(MPSP)

extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr);
Eterm erts_match_set_lint(Process *p, Eterm matchexpr); 
extern void erts_match_set_release_result(Process* p);

enum erts_pam_run_flags {
    ERTS_PAM_TMP_RESULT=1,
    ERTS_PAM_COPY_RESULT=2,
    ERTS_PAM_CONTIGUOUS_TUPLE=4,
    ERTS_PAM_IGNORE_TRACE_SILENT=8
};
extern Eterm erts_match_set_run(Process *p, Binary *mpsp, 
				Eterm *args, int num_args,
				enum erts_pam_run_flags in_flags,
				Uint32 *return_flags);
extern Eterm erts_match_set_get_source(Binary *mpsp);
extern void erts_match_prog_foreach_offheap(Binary *b,
					    void (*)(ErlOffHeap *, void *),
					    void *);

#define MATCH_SET_RETURN_TRACE    (0x1) /* return trace requested */
#define MATCH_SET_RETURN_TO_TRACE (0x2) /* Misleading name, it is not actually
					   set by the match program, but by the
					   breakpoint functions */
#define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */
#define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE)

extern erts_driver_t vanilla_driver;
extern erts_driver_t spawn_driver;
extern erts_driver_t fd_driver;

int erts_beam_jump_table(void);

/* Should maybe be placed in erl_message.h, but then we get an include mess. */
ERTS_GLB_INLINE Eterm *
erts_alloc_message_heap_state(Uint size,
			      ErlHeapFragment **bpp,
			      ErlOffHeap **ohpp,
			      Process *receiver,
			      ErtsProcLocks *receiver_locks,
			      erts_aint32_t *statep);

ERTS_GLB_INLINE Eterm *
erts_alloc_message_heap(Uint size,
			ErlHeapFragment **bpp,
			ErlOffHeap **ohpp,
			Process *receiver,
			ErtsProcLocks *receiver_locks);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

/*
 * NOTE: erts_alloc_message_heap() releases msg q and status
 *       lock on receiver without ensuring that other locks are
 *       held. User is responsible to ensure that the receiver
 *       pointer cannot become invalid until after message has
 *       been passed. This is normal done either by increasing
 *       reference count on process (preferred) or by holding
 *       main or link lock over the whole message passing
 *       operation.
 */

ERTS_GLB_INLINE Eterm *
erts_alloc_message_heap_state(Uint size,
			      ErlHeapFragment **bpp,
			      ErlOffHeap **ohpp,
			      Process *receiver,
			      ErtsProcLocks *receiver_locks,
			      erts_aint32_t *statep)
{
    Eterm *hp;
    erts_aint32_t state;
#ifdef ERTS_SMP
    int locked_main = 0;
    state = erts_smp_atomic32_read_acqb(&receiver->state);
    if (statep)
	*statep = state;
    if (state & (ERTS_PSFLG_EXITING
		 | ERTS_PSFLG_PENDING_EXIT))
	goto allocate_in_mbuf;
#endif
# 1399 "beam/global.h"

    if (size > (Uint) INT_MAX)
	erts_exit(ERTS_ABORT_EXIT, "HUGE size (%beu)\n", size);

    if (
#if defined(ERTS_SMP)
	*receiver_locks & ERTS_PROC_LOCK_MAIN
#else
# 1407 "beam/global.h"
	1
#endif
# 1409 "beam/global.h"
	) {
#ifdef ERTS_SMP
    try_allocate_on_heap:
#endif
# 1413 "beam/global.h"
	state = erts_smp_atomic32_read_nob(&receiver->state);
	if (statep)
	    *statep = state;
	if ((state & (ERTS_PSFLG_EXITING
		      | ERTS_PSFLG_PENDING_EXIT))
	    || (receiver->flags & F_DISABLE_GC)
	    || HEAP_LIMIT(receiver) - HEAP_TOP(receiver) <= size) {
	    /*
	     * The heap is either potentially in an inconsistent
	     * state, or not large enough.
	     */
#ifdef ERTS_SMP
	    if (locked_main) {
		*receiver_locks &= ~ERTS_PROC_LOCK_MAIN;
		erts_smp_proc_unlock(receiver, ERTS_PROC_LOCK_MAIN);
	    }
#endif
# 1430 "beam/global.h"
	    goto allocate_in_mbuf;
	}
	hp = HEAP_TOP(receiver);
	HEAP_TOP(receiver) = hp + size;
	*bpp = NULL;
	*ohpp = &MSO(receiver);
    }
#ifdef ERTS_SMP
    else if (erts_smp_proc_trylock(receiver, ERTS_PROC_LOCK_MAIN) == 0) {
	locked_main = 1;
	*receiver_locks |= ERTS_PROC_LOCK_MAIN;
	goto try_allocate_on_heap;
    }
#endif
# 1444 "beam/global.h"
    else {
	ErlHeapFragment *bp;
    allocate_in_mbuf:
	bp = new_message_buffer(size);
	hp = bp->mem;
	*bpp = bp;
	*ohpp = &bp->off_heap;
    }

    return hp;
}

ERTS_GLB_INLINE Eterm *
erts_alloc_message_heap(Uint size,
			ErlHeapFragment **bpp,
			ErlOffHeap **ohpp,
			Process *receiver,
			ErtsProcLocks *receiver_locks)
{
    return erts_alloc_message_heap_state(size, bpp, ohpp, receiver,
					 receiver_locks, NULL);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1468 "beam/global.h"

#if !HEAP_ON_C_STACK
#  if defined(DEBUG)
#    define DeclareTmpHeap(VariableName,Size,Process) \
       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,Process)
#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
      Type *VariableName = (Type *) erts_debug_allocate_tmp_heap(sizeof(Type)/sizeof(Eterm),Process)
#    define DeclareTmpHeapNoproc(VariableName,Size) \
       Eterm *VariableName = erts_debug_allocate_tmp_heap(Size,NULL)
#    define UseTmpHeap(Size,Proc) \
       do { \
         erts_debug_use_tmp_heap((Size),(Proc)); \
       } while (0)
#    define UnUseTmpHeap(Size,Proc) \
       do { \
         erts_debug_unuse_tmp_heap((Size),(Proc)); \
       } while (0)
#    define UseTmpHeapNoproc(Size) \
       do { \
         erts_debug_use_tmp_heap(Size,NULL); \
       } while (0)
#    define UnUseTmpHeapNoproc(Size) \
       do { \
         erts_debug_unuse_tmp_heap(Size,NULL); \
       } while (0)
#  else
# 1494 "beam/global.h"
#    define DeclareTmpHeap(VariableName,Size,Process) \
       Eterm *VariableName = (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
#    define DeclareTypedTmpHeap(Type,VariableName,Process)		\
      Type *VariableName = (Type *) (ERTS_PROC_GET_SCHDATA(Process)->tmp_heap)+(ERTS_PROC_GET_SCHDATA(Process)->num_tmp_heap_used)
#    define DeclareTmpHeapNoproc(VariableName,Size) \
       Eterm *VariableName = (erts_get_scheduler_data()->tmp_heap)+(erts_get_scheduler_data()->num_tmp_heap_used)
#    define UseTmpHeap(Size,Proc) \
       do { \
         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used += (Size); \
       } while (0)
#    define UnUseTmpHeap(Size,Proc) \
       do { \
         ERTS_PROC_GET_SCHDATA(Proc)->num_tmp_heap_used -= (Size); \
       } while (0)
#    define UseTmpHeapNoproc(Size) \
       do { \
         erts_get_scheduler_data()->num_tmp_heap_used += (Size); \
       } while (0)
#    define UnUseTmpHeapNoproc(Size) \
       do { \
         erts_get_scheduler_data()->num_tmp_heap_used -= (Size); \
       } while (0)


#  endif
# 1519 "beam/global.h"

#else
# 1521 "beam/global.h"
#  define DeclareTmpHeap(VariableName,Size,Process) \
     Eterm VariableName[Size]
#  define DeclareTypedTmpHeap(Type,VariableName,Process)	\
     Type VariableName[1]
#  define DeclareTmpHeapNoproc(VariableName,Size) \
     Eterm VariableName[Size]
#  define UseTmpHeap(Size,Proc) /* Nothing */
#  define UnUseTmpHeap(Size,Proc) /* Nothing */
#  define UseTmpHeapNoproc(Size) /* Nothing */
#  define UnUseTmpHeapNoproc(Size) /* Nothing */
#endif /* HEAP_ON_C_STACK */
# 1532 "beam/global.h"

ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf);
ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf);
ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf);
ERTS_GLB_INLINE void dtrace_fun_decode(Process *process,
				       Eterm module, Eterm function, int arity,
				       char *process_buf, char *mfa_buf);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

#if 0 /* expanded by -frewrite-includes */
#include "dtrace-wrapper.h"
#endif /* expanded by -frewrite-includes */
# 1542 "beam/global.h"
# 1 "beam/dtrace-wrapper.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Dustin Sallings, Michal Ptaszek, Scott Lystig Fritchie 2011-2012.
 * All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __DTRACE_WRAPPER_H
#define __DTRACE_WRAPPER_H

#define DTRACE_TERM_BUF_SIZE 256

/*
 * Some varieties of SystemTap macros do not like statically-sized
 * char[N] buffers.  (For example, CentOS 6's macros.)
 * So, we'll play a game to humor them.
 *
 * The code necessary to play nice with CentOS 6's SystemTap looks
 * stupid to a C programmer's eyes, so we hide the ugliness with this
 * macro, which expands:
 *
 *    DTRACE_CHARBUF(proc_name, 64);
 *
 * to become:
 *
 *    char proc_name_BUFFER[64], *proc_name = proc_name_BUFFER;
 */

#define DTRACE_CHARBUF(name, size) \
    char name##_BUFFER[size], *name = name##_BUFFER

#define DTRACE_CHARBUF_NAME(name) name##_BUFFER

#if defined(USE_DYNAMIC_TRACE) && defined(USE_VM_PROBES) 

#if 0 /* expanded by -frewrite-includes */
#include "erlang_dtrace.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/dtrace-wrapper.h"
# 51 "beam/dtrace-wrapper.h"

#define DTRACE_ENABLED(name)                         \
    erlang_##name##_enabled()
#define DTRACE0(name)                                \
    erlang_##name()
#define DTRACE1(name, a0)                            \
    erlang_##name(a0)
#define DTRACE2(name, a0, a1)                        \
    erlang_##name((a0), (a1))
#define DTRACE3(name, a0, a1, a2)                    \
    erlang_##name((a0), (a1), (a2))
#define DTRACE4(name, a0, a1, a2, a3)                \
    erlang_##name((a0), (a1), (a2), (a3))
#define DTRACE5(name, a0, a1, a2, a3, a4)            \
    erlang_##name((a0), (a1), (a2), (a3), (a4))
#define DTRACE6(name, a0, a1, a2, a3, a4, a5)        \
    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5))
#define DTRACE7(name, a0, a1, a2, a3, a4, a5, a6)    \
    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6))
#define DTRACE10(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \
    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7), (a8), (a9))
#define DTRACE11(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \
    erlang_##name((a0), (a1), (a2), (a3), (a4), (a5), (a6), (a7), (a8), (a9), (a10))

#if defined(_SDT_PROBE) && !defined(STAP_PROBE11)
/* SLF: This is Ubuntu 11-style SystemTap hackery */
/* work arround for missing STAP macro */
#define STAP_PROBE11(provider,name,arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \
  _SDT_PROBE(provider, name, 11, \
             (arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11))
#define _SDT_ASM_OPERANDS_11(arg1,arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10,arg11) \
  _SDT_ASM_OPERANDS_10(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9,arg10), \
    _SDT_ARG(11, arg11)
#endif
# 85 "beam/dtrace-wrapper.h"

#ifdef STAP_PROBE_ADDR
/* SLF: This is CentOS 5-style SystemTap hackery */
/* SystemTap compat mode cannot support 11 args. We'll ignore the 11th */
#define STAP_PROBE11(provider,probe,parm1,parm2,parm3,parm4,parm5,parm6,parm7,parm8,parm9,parm10,parm11) \
    STAP_PROBE10(provider,probe,(parm1),(parm2),(parm3),(parm4),(parm5),(parm6),(parm7),(parm8),(parm9),(parm10))
#endif /* STAP_PROBE_ADDR */
# 92 "beam/dtrace-wrapper.h"

#else   /* USE_DYNAMIC_TRACE && USE_VM_PROBES */
# 94 "beam/dtrace-wrapper.h"

/* Render all macros to do nothing */
#define DTRACE_ENABLED(name)                         0
#define DTRACE0(name)                                do {} while (0)
#define DTRACE1(name, a0)                            do {} while (0)
#define DTRACE2(name, a0, a1)                        do {} while (0)
#define DTRACE3(name, a0, a1, a2)                    do {} while (0)
#define DTRACE4(name, a0, a1, a2, a3)                do {} while (0)
#define DTRACE5(name, a0, a1, a2, a3, a4)            do {} while (0)
#define DTRACE6(name, a0, a1, a2, a3, a4, a5)        do {} while (0)
#define DTRACE7(name, a0, a1, a2, a3, a4, a5, a6)    do {} while (0)
#define DTRACE10(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) \
                                                     do {} while (0)
#define DTRACE11(name, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) \
                                                     do {} while (0)

#endif  /* USE_DYNAMIC_TRACE && USE_VM_PROBES */
# 111 "beam/dtrace-wrapper.h"

#endif  /* __DTRACE_WRAPPER_H */
# 113 "beam/dtrace-wrapper.h"
# 1543 "beam/global.h" 2

ERTS_GLB_INLINE void
dtrace_pid_str(Eterm pid, char *process_buf)
{
    erts_snprintf(process_buf, DTRACE_TERM_BUF_SIZE, "<%lu.%lu.%lu>",
                  pid_channel_no(pid),
                  pid_number(pid),
                  pid_serial(pid));
}

ERTS_GLB_INLINE void
dtrace_proc_str(Process *process, char *process_buf)
{
    dtrace_pid_str(process->common.id, process_buf);
}

ERTS_GLB_INLINE void
dtrace_port_str(Port *port, char *port_buf)
{
    erts_snprintf(port_buf, DTRACE_TERM_BUF_SIZE, "#Port<%lu.%lu>",
                  port_channel_no(port->common.id),
                  port_number(port->common.id));
}

ERTS_GLB_INLINE void
dtrace_fun_decode(Process *process,
                  Eterm module, Eterm function, int arity,
                  char *process_buf, char *mfa_buf)
{
    if (process_buf) {
        dtrace_proc_str(process, process_buf);
    }

    erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d",
                  module, function, arity);
}
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1580 "beam/global.h"

#endif /* !__GLOBAL_H__ */
# 1582 "beam/global.h"
# 29 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/beam_emu.c"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 30 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "error.h"
#endif /* expanded by -frewrite-includes */
# 30 "beam/beam_emu.c"
# 31 "beam/beam_emu.c"
#if 0 /* expanded by -frewrite-includes */
#include "bif.h"
#endif /* expanded by -frewrite-includes */
# 31 "beam/beam_emu.c"
# 1 "beam/bif.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __BIF_H__
#define __BIF_H__

extern Export *erts_await_result;
extern Export* erts_format_cpu_topology_trap;
extern Export *erts_convert_time_unit_trap;

#define BIF_RETTYPE Eterm

#define BIF_P A__p

#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS
#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS
#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS
#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS
#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS

#define BIF_ARG_1  (BIF__ARGS[0])
#define BIF_ARG_2  (BIF__ARGS[1])
#define BIF_ARG_3  (BIF__ARGS[2])
#define BIF_ARG_4  (BIF__ARGS[3])

#define ERTS_IS_PROC_OUT_OF_REDS(p)		\
    ((p)->fcalls > 0				\
     ? 0					\
     : (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))	\
	? (p)->fcalls == 0			\
	: ((p)->fcalls == -CONTEXT_REDS)))

#define BUMP_ALL_REDS(p) do {			\
    if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) 	\
	(p)->fcalls = 0; 			\
    else 					\
	(p)->fcalls = -CONTEXT_REDS;		\
} while(0)


#define ERTS_VBUMP_ALL_REDS(p)						\
do {									\
    if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) {				\
	if ((p)->fcalls > 0)						\
	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls;	\
	(p)->fcalls = 0;						\
    }									\
    else {								\
	if ((p)->fcalls > -CONTEXT_REDS)				\
	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds			\
		+= ((p)->fcalls - (-CONTEXT_REDS));			\
	(p)->fcalls = -CONTEXT_REDS;					\
    }									\
} while(0)

#define BUMP_REDS(p, gc) do {			   \
     ASSERT(p);		 			   \
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));\
     (p)->fcalls -= (gc); 			   \
     if ((p)->fcalls < 0) { 			   \
	if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p)))   \
           (p)->fcalls = 0; 			   \
	else if ((p)->fcalls < -CONTEXT_REDS)      \
           (p)->fcalls = -CONTEXT_REDS; 	   \
     } 						   \
} while(0)


#define ERTS_VBUMP_REDS(p, reds)					\
do {									\
    if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) {				\
	if ((p)->fcalls >= reds) {					\
	    (p)->fcalls -= reds;					\
	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += reds;		\
	}								\
	else {								\
	    if ((p)->fcalls > 0)					\
		ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls;\
	    (p)->fcalls = 0;						\
	}								\
    }									\
    else {								\
	if ((p)->fcalls >= reds - CONTEXT_REDS) {			\
	    (p)->fcalls -= reds;					\
	    ERTS_PROC_GET_SCHDATA((p))->virtual_reds += reds;		\
	}								\
	else {								\
	    if ((p)->fcalls > -CONTEXT_REDS)				\
		ERTS_PROC_GET_SCHDATA((p))->virtual_reds		\
		    += (p)->fcalls - (-CONTEXT_REDS);			\
	    (p)->fcalls = -CONTEXT_REDS;				\
	}								\
    }									\
} while(0)

#define ERTS_BIF_REDS_LEFT(p)						\
  (ERTS_PROC_GET_SAVED_CALLS_BUF((p))					\
   ? ((p)->fcalls > -CONTEXT_REDS ? ((p)->fcalls - (-CONTEXT_REDS)) : 0)\
   : ((p)->fcalls > 0 ? (p)->fcalls : 0))

#define BIF_RET2(x, gc) do {			\
    BUMP_REDS(BIF_P, (gc));			\
    return (x);					\
} while(0)

#define BIF_RET(x) return (x)

#define ERTS_BIF_PREP_RET(Ret, Val) ((Ret) = (Val))

#define BIF_ERROR(p,r) do { 			\
    (p)->freason = r; 				\
    return THE_NON_VALUE; 			\
} while(0)

#define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif)		\
do {								\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    return THE_NON_VALUE; 					\
} while (0)

#define ERTS_BIF_ERROR_TRAPPED1(Proc, Reason, Bif, A0)		\
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    reg[0] = (Eterm) (A0);					\
    return THE_NON_VALUE; 					\
} while (0)

#define ERTS_BIF_ERROR_TRAPPED2(Proc, Reason, Bif, A0, A1)	\
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    reg[0] = (Eterm) (A0);					\
    reg[1] = (Eterm) (A1);					\
    return THE_NON_VALUE; 					\
} while (0)

#define ERTS_BIF_ERROR_TRAPPED3(Proc, Reason, Bif, A0, A1, A2)	\
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    reg[0] = (Eterm) (A0);					\
    reg[1] = (Eterm) (A1);					\
    reg[2] = (Eterm) (A2);					\
    return THE_NON_VALUE; 					\
} while (0)

#define ERTS_BIF_PREP_ERROR(Ret, Proc, Reason)	\
do {						\
    (Proc)->freason = (Reason);			\
    (Ret) = THE_NON_VALUE;			\
} while (0)

#define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif)	\
do {								\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_ERROR_TRAPPED1(Ret, Proc, Reason, Bif, A0) \
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    reg[0] = (Eterm) (A0);					\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_ERROR_TRAPPED2(Ret, Proc, Reason, Bif, A0, A1) \
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    reg[0] = (Eterm) (A0);					\
    reg[1] = (Eterm) (A1);					\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_ERROR_TRAPPED3(Ret, Proc, Reason, Bif, A0, A1, A2) \
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->freason = (Reason);					\
    (Proc)->current = (Bif)->code;				\
    reg[0] = (Eterm) (A0);					\
    reg[1] = (Eterm) (A1);					\
    reg[2] = (Eterm) (A2);					\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_TRAP0(Ret, Trap, Proc)	\
do {						\
    (Proc)->arity = 0;				\
    (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]);	\
    (Proc)->freason = TRAP;			\
    (Ret) = THE_NON_VALUE;			\
} while (0)

#define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0)		\
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->arity = 1;						\
    reg[0] = (Eterm) (A0);					\
    (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
    (Proc)->freason = TRAP;					\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1)		\
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->arity = 2;						\
    reg[0] = (Eterm) (A0);					\
    reg[1] = (Eterm) (A1);					\
    (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
    (Proc)->freason = TRAP;					\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2)	\
do {								\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->arity = 3;						\
    reg[0] = (Eterm) (A0);					\
    reg[1] = (Eterm) (A1);					\
    reg[2] = (Eterm) (A2);					\
    (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
    (Proc)->freason = TRAP;					\
    (Ret) = THE_NON_VALUE;					\
} while (0)

#define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\
do {							\
    Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array;	\
    (Proc)->arity = 3;					\
    reg[0] = (Eterm) (A0);		\
    reg[1] = (Eterm) (A1);		\
    reg[2] = (Eterm) (A2);		\
    (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \
    (Proc)->freason = TRAP;				\
} while (0)

#define BIF_TRAP0(p, Trap_) do {		\
      (p)->arity = 0;				\
      (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]);	\
      (p)->freason = TRAP;			\
      return THE_NON_VALUE;			\
 } while(0)

#define BIF_TRAP1(Trap_, p, A0) do {				\
      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
      (p)->arity = 1;						\
      reg[0] = (A0);						\
      (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
      (p)->freason = TRAP;					\
      return THE_NON_VALUE;					\
 } while(0)

#define BIF_TRAP2(Trap_, p, A0, A1) do {			\
      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
      (p)->arity = 2;						\
      reg[0] = (A0);						\
      reg[1] = (A1);						\
      (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
      (p)->freason = TRAP;					\
      return THE_NON_VALUE;					\
 } while(0)

#define BIF_TRAP3(Trap_, p, A0, A1, A2) do {			\
      Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array;	\
      (p)->arity = 3;						\
      reg[0] = (A0);						\
      reg[1] = (A1);						\
      reg[2] = (A2);						\
      (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \
      (p)->freason = TRAP;					\
      return THE_NON_VALUE;					\
 } while(0)

#define BIF_TRAP_CODE_PTR_0(p, Code_) do {	\
      (p)->arity = 0;				\
      (p)->i = (BeamInstr*) (Code_);		\
      (p)->freason = TRAP;			\
      return THE_NON_VALUE;			\
 } while(0)

#define BIF_TRAP_CODE_PTR_(p, Code_) do {	\
      (p)-> i = (BeamInstr*) (Code_);		\
      (p)->freason = TRAP;			\
      return THE_NON_VALUE;			\
 } while(0)

extern Export bif_return_trap_export;
#ifdef DEBUG
#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL)		\
do {									\
    ERTS_VBUMP_ALL_REDS(P);						\
    ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL),	\
			(DEBUG_VAL));					\
} while (0)
#else
# 323 "beam/bif.h"
#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL)		\
do {									\
    ERTS_VBUMP_ALL_REDS(P);						\
    ERTS_BIF_PREP_TRAP1(RET, &bif_return_trap_export, (P), (VAL));	\
} while (0)
#endif
# 329 "beam/bif.h"

#define ERTS_BIF_PREP_YIELD_RETURN(RET, P, VAL) \
  ERTS_BIF_PREP_YIELD_RETURN_X(RET, (P), (VAL), am_undefined)

#ifdef DEBUG
#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL)			\
do {									\
    ERTS_VBUMP_ALL_REDS(P);						\
    BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (DEBUG_VAL));	\
} while (0)
#else
# 340 "beam/bif.h"
#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL)			\
do {									\
    ERTS_VBUMP_ALL_REDS(P);						\
    BIF_TRAP1(&bif_return_trap_export, (P), (VAL));			\
} while (0)
#endif
# 346 "beam/bif.h"

#define ERTS_BIF_RETURN_YIELD(P) ERTS_VBUMP_ALL_REDS((P))

#define ERTS_BIF_YIELD_RETURN(P, VAL) \
  ERTS_BIF_YIELD_RETURN_X((P), (VAL), am_undefined)

#define ERTS_BIF_PREP_YIELD0(RET, TRP, P)				\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    ERTS_BIF_PREP_TRAP0(RET, (TRP), (P));				\
} while (0)

#define ERTS_BIF_PREP_YIELD1(RET, TRP, P, A0)				\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    ERTS_BIF_PREP_TRAP1(RET, (TRP), (P), (A0));				\
} while (0)

#define ERTS_BIF_PREP_YIELD2(RET, TRP, P, A0, A1)			\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    ERTS_BIF_PREP_TRAP2(RET, (TRP), (P), (A0), (A1));			\
} while (0)

#define ERTS_BIF_PREP_YIELD3(RET, TRP, P, A0, A1, A2)			\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    ERTS_BIF_PREP_TRAP3(RET, (TRP), (P), (A0), (A1), (A2));		\
} while (0)

#define ERTS_BIF_YIELD0(TRP, P)						\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    BIF_TRAP0((TRP), (P));						\
} while (0)

#define ERTS_BIF_YIELD1(TRP, P, A0)					\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    BIF_TRAP1((TRP), (P), (A0));					\
} while (0)

#define ERTS_BIF_YIELD2(TRP, P, A0, A1)					\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    BIF_TRAP2((TRP), (P), (A0), (A1));					\
} while (0)

#define ERTS_BIF_YIELD3(TRP, P, A0, A1, A2)				\
do {									\
    ERTS_VBUMP_ALL_REDS((P));						\
    BIF_TRAP3((TRP), (P), (A0), (A1), (A2));				\
} while (0)

#define ERTS_BIF_EXITED(PROC)		\
do {					\
    KILL_CATCHES((PROC));		\
    BIF_ERROR((PROC), EXC_EXIT);	\
} while (0)

#define ERTS_BIF_CHK_EXITED(PROC)	\
do {					\
    if (ERTS_PROC_IS_EXITING((PROC)))	\
	ERTS_BIF_EXITED((PROC));	\
} while (0)

/*
 * The ERTS_BIF_*_AWAIT_X_*_TRAP makros either exits the caller, or
 * sets up a trap to erlang:await_proc_exit/3.
 *
 * The caller is acquired to hold the 'main' lock on C_P. No other locks
 * are allowed to be held.
 */

#define ERTS_BIF_PREP_AWAIT_X_DATA_TRAP(RET, C_P, PID, DATA)		\
do {									\
    erts_bif_prep_await_proc_exit_data_trap((C_P), (PID), (DATA));	\
    (RET) = THE_NON_VALUE;						\
} while (0)

#define ERTS_BIF_PREP_AWAIT_X_REASON_TRAP(RET, C_P, PID)		\
do {									\
    erts_bif_prep_await_proc_exit_reason_trap((C_P), (PID));		\
    (RET) = THE_NON_VALUE;						\
} while (0)

#define ERTS_BIF_PREP_AWAIT_X_APPLY_TRAP(RET, C_P, PID, M, F, A, AN)	\
do {									\
    erts_bif_prep_await_proc_exit_apply_trap((C_P), (PID),		\
					     (M), (F), (A), (AN));	\
    (RET) = THE_NON_VALUE;						\
} while (0)

#define ERTS_BIF_AWAIT_X_DATA_TRAP(C_P, PID, DATA)			\
do {									\
    erts_bif_prep_await_proc_exit_data_trap((C_P), (PID), (DATA));	\
    return THE_NON_VALUE;						\
} while (0)

#define ERTS_BIF_AWAIT_X_REASON_TRAP(C_P, PID)				\
do {									\
    erts_bif_prep_await_proc_exit_reason_trap((C_P), (PID));		\
    return THE_NON_VALUE;						\
} while (0)

#define ERTS_BIF_AWAIT_X_APPLY_TRAP(C_P, PID, M, F, A, AN)		\
do {									\
    erts_bif_prep_await_proc_exit_apply_trap((C_P), (PID),		\
					     (M), (F), (A), (AN));	\
    return THE_NON_VALUE;						\
} while (0)

void
erts_bif_prep_await_proc_exit_data_trap(Process *c_p,
					Eterm pid,
					Eterm data);
void
erts_bif_prep_await_proc_exit_reason_trap(Process *c_p,
					  Eterm pid);
void
erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
					 Eterm pid,
					 Eterm module,
					 Eterm function,
					 Eterm args[],
					 int nargs);

#ifdef ERL_WANT_HIPE_BIF_WRAPPER__

#ifndef HIPE

#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)

#else
# 480 "beam/bif.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_fun.h"
#endif /* expanded by -frewrite-includes */
# 481 "beam/bif.h"
# 482 "beam/bif.h"
#if 0 /* expanded by -frewrite-includes */
#include "hipe_mode_switch.h"
#endif /* expanded by -frewrite-includes */
# 482 "beam/bif.h"
# 483 "beam/bif.h"

/*
 * Hipe wrappers used by native code for BIFs that disable GC while trapping.
 * Also add usage of the wrapper in ../hipe/hipe_bif_list.m4
 *
 * Problem:
 * When native code calls a BIF that traps, hipe_mode_switch will push a
 * "trap frame" on the Erlang stack in order to find its way back from beam_emu
 * back to native caller when finally done. If GC is disabled and stack/heap
 * is full there is no place to push the "trap frame".
 *
 * Solution:
 * We reserve space on stack for the "trap frame" here before the BIF is called.
 * If the BIF does not trap, the space is reclaimed here before returning.
 * If the BIF traps, hipe_push_beam_trap_frame() will detect that a "trap frame"
 * already is reserved and use it.
 */


#define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)			\
BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
						     Eterm* args);	\
BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
						     Eterm* args)	\
{									\
    BIF_RETTYPE  res;							\
    hipe_reserve_beam_trap_frame(c_p, args, ARITY);			\
    res =  BIF_NAME ## _ ## ARITY (c_p, args);				\
    if (is_value(res) || c_p->freason != TRAP) {			\
	hipe_unreserve_beam_trap_frame(c_p);				\
    }									\
    return res;								\
}

#endif
# 518 "beam/bif.h"

#endif /* ERL_WANT_HIPE_BIF_WRAPPER__ */
# 520 "beam/bif.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_bif_table.h"
#endif /* expanded by -frewrite-includes */
# 521 "beam/bif.h"
# 1 "armv6-portbld-freebsd11.0/erl_bif_table.h" 1
/*
 *  Warning: Do not edit this file.  It was automatically
 *  generated by 'make_tables' on Thu May 12 14:27:46 2016.
 */

#ifndef __ERL_BIF_TABLE_H__
#define __ERL_BIF_TABLE_H__
typedef void *BifFunction;

typedef struct bif_entry {
    Eterm module;
    Eterm name;
    int arity;
    BifFunction f;
    BifFunction traced;
} BifEntry;

extern BifEntry bif_table[];
extern Export* bif_export[];

#define BIF_SIZE 508

#define BIF_abs_1 0
#define BIF_adler32_1 1
#define BIF_adler32_2 2
#define BIF_adler32_combine_3 3
#define BIF_apply_3 4
#define BIF_atom_to_list_1 5
#define BIF_binary_to_list_1 6
#define BIF_binary_to_list_3 7
#define BIF_binary_to_term_1 8
#define BIF_crc32_1 9
#define BIF_crc32_2 10
#define BIF_crc32_combine_3 11
#define BIF_date_0 12
#define BIF_delete_module_1 13
#define BIF_display_1 14
#define BIF_display_string_1 15
#define BIF_display_nl_0 16
#define BIF_element_2 17
#define BIF_erase_0 18
#define BIF_erase_1 19
#define BIF_exit_1 20
#define BIF_exit_2 21
#define BIF_external_size_1 22
#define BIF_external_size_2 23
#define BIF_float_1 24
#define BIF_float_to_list_1 25
#define BIF_float_to_list_2 26
#define BIF_fun_info_2 27
#define BIF_garbage_collect_0 28
#define BIF_get_0 29
#define BIF_get_1 30
#define BIF_get_keys_1 31
#define BIF_group_leader_0 32
#define BIF_group_leader_2 33
#define BIF_halt_0 34
#define BIF_halt_1 35
#define BIF_halt_2 36
#define BIF_phash_2 37
#define BIF_phash2_1 38
#define BIF_phash2_2 39
#define BIF_hd_1 40
#define BIF_integer_to_list_1 41
#define BIF_is_alive_0 42
#define BIF_length_1 43
#define BIF_link_1 44
#define BIF_list_to_atom_1 45
#define BIF_list_to_binary_1 46
#define BIF_list_to_float_1 47
#define BIF_list_to_integer_1 48
#define BIF_list_to_pid_1 49
#define BIF_list_to_tuple_1 50
#define BIF_loaded_0 51
#define BIF_localtime_0 52
#define BIF_localtime_to_universaltime_2 53
#define BIF_make_ref_0 54
#define BIF_unique_integer_0 55
#define BIF_unique_integer_1 56
#define BIF_md5_1 57
#define BIF_md5_init_0 58
#define BIF_md5_update_2 59
#define BIF_md5_final_1 60
#define BIF_module_loaded_1 61
#define BIF_function_exported_3 62
#define BIF_monitor_node_2 63
#define BIF_monitor_node_3 64
#define BIF_node_1 65
#define BIF_node_0 66
#define BIF_nodes_1 67
#define BIF_now_0 68
#define BIF_monotonic_time_0 69
#define BIF_monotonic_time_1 70
#define BIF_system_time_0 71
#define BIF_system_time_1 72
#define BIF_time_offset_0 73
#define BIF_time_offset_1 74
#define BIF_timestamp_0 75
#define BIF_open_port_2 76
#define BIF_pid_to_list_1 77
#define BIF_ports_0 78
#define BIF_pre_loaded_0 79
#define BIF_process_flag_2 80
#define BIF_process_flag_3 81
#define BIF_process_info_1 82
#define BIF_process_info_2 83
#define BIF_processes_0 84
#define BIF_purge_module_1 85
#define BIF_put_2 86
#define BIF_register_2 87
#define BIF_registered_0 88
#define BIF_round_1 89
#define BIF_self_0 90
#define BIF_setelement_3 91
#define BIF_size_1 92
#define BIF_spawn_3 93
#define BIF_spawn_link_3 94
#define BIF_split_binary_2 95
#define BIF_statistics_1 96
#define BIF_term_to_binary_1 97
#define BIF_term_to_binary_2 98
#define BIF_throw_1 99
#define BIF_time_0 100
#define BIF_tl_1 101
#define BIF_trunc_1 102
#define BIF_tuple_to_list_1 103
#define BIF_universaltime_0 104
#define BIF_universaltime_to_localtime_1 105
#define BIF_unlink_1 106
#define BIF_unregister_1 107
#define BIF_whereis_1 108
#define BIF_spawn_opt_1 109
#define BIF_setnode_2 110
#define BIF_setnode_3 111
#define BIF_dist_exit_3 112
#define BIF_erts_internal_port_info_1 113
#define BIF_erts_internal_port_info_2 114
#define BIF_erts_internal_port_call_3 115
#define BIF_erts_internal_port_command_3 116
#define BIF_erts_internal_port_control_3 117
#define BIF_erts_internal_port_close_1 118
#define BIF_erts_internal_port_connect_2 119
#define BIF_erts_internal_request_system_task_3 120
#define BIF_erts_internal_check_process_code_2 121
#define BIF_erts_internal_map_to_tuple_keys_1 122
#define BIF_erts_internal_map_type_1 123
#define BIF_erts_internal_map_hashmap_children_1 124
#define BIF_erts_internal_time_unit_0 125
#define BIF_erts_internal_is_system_process_1 126
#define BIF_erts_internal_system_check_1 127
#define BIF_port_set_data_2 128
#define BIF_port_get_data_1 129
#define BIF_trace_pattern_2 130
#define BIF_trace_pattern_3 131
#define BIF_trace_3 132
#define BIF_trace_info_2 133
#define BIF_trace_delivered_1 134
#define BIF_seq_trace_2 135
#define BIF_seq_trace_info_1 136
#define BIF_seq_trace_print_1 137
#define BIF_seq_trace_print_2 138
#define BIF_suspend_process_2 139
#define BIF_resume_process_1 140
#define BIF_process_display_2 141
#define BIF_bump_reductions_1 142
#define BIF_math_cos_1 143
#define BIF_math_cosh_1 144
#define BIF_math_sin_1 145
#define BIF_math_sinh_1 146
#define BIF_math_tan_1 147
#define BIF_math_tanh_1 148
#define BIF_math_acos_1 149
#define BIF_math_acosh_1 150
#define BIF_math_asin_1 151
#define BIF_math_asinh_1 152
#define BIF_math_atan_1 153
#define BIF_math_atanh_1 154
#define BIF_math_erf_1 155
#define BIF_math_erfc_1 156
#define BIF_math_exp_1 157
#define BIF_math_log_1 158
#define BIF_math_log2_1 159
#define BIF_math_log10_1 160
#define BIF_math_sqrt_1 161
#define BIF_math_atan2_2 162
#define BIF_math_pow_2 163
#define BIF_start_timer_3 164
#define BIF_start_timer_4 165
#define BIF_send_after_3 166
#define BIF_send_after_4 167
#define BIF_cancel_timer_1 168
#define BIF_cancel_timer_2 169
#define BIF_read_timer_1 170
#define BIF_read_timer_2 171
#define BIF_make_tuple_2 172
#define BIF_append_element_2 173
#define BIF_make_tuple_3 174
#define BIF_system_flag_2 175
#define BIF_system_info_1 176
#define BIF_system_monitor_0 177
#define BIF_system_monitor_1 178
#define BIF_system_monitor_2 179
#define BIF_system_profile_2 180
#define BIF_system_profile_0 181
#define BIF_ref_to_list_1 182
#define BIF_port_to_list_1 183
#define BIF_fun_to_list_1 184
#define BIF_monitor_2 185
#define BIF_demonitor_1 186
#define BIF_demonitor_2 187
#define BIF_is_process_alive_1 188
#define BIF_error_1 189
#define BIF_error_2 190
#define BIF_raise_3 191
#define BIF_get_stacktrace_0 192
#define BIF_is_builtin_3 193
#define BIF_and_2 194
#define BIF_or_2 195
#define BIF_xor_2 196
#define BIF_not_1 197
#define BIF_sgt_2 198
#define BIF_sge_2 199
#define BIF_slt_2 200
#define BIF_sle_2 201
#define BIF_seq_2 202
#define BIF_seqeq_2 203
#define BIF_sneq_2 204
#define BIF_sneqeq_2 205
#define BIF_splus_2 206
#define BIF_sminus_2 207
#define BIF_stimes_2 208
#define BIF_div_2 209
#define BIF_intdiv_2 210
#define BIF_rem_2 211
#define BIF_bor_2 212
#define BIF_band_2 213
#define BIF_bxor_2 214
#define BIF_bsl_2 215
#define BIF_bsr_2 216
#define BIF_bnot_1 217
#define BIF_sminus_1 218
#define BIF_splus_1 219
#define BIF_ebif_bang_2 220
#define BIF_send_2 221
#define BIF_send_3 222
#define BIF_ebif_plusplus_2 223
#define BIF_append_2 224
#define BIF_ebif_minusminus_2 225
#define BIF_subtract_2 226
#define BIF_is_atom_1 227
#define BIF_is_list_1 228
#define BIF_is_tuple_1 229
#define BIF_is_float_1 230
#define BIF_is_integer_1 231
#define BIF_is_number_1 232
#define BIF_is_pid_1 233
#define BIF_is_port_1 234
#define BIF_is_reference_1 235
#define BIF_is_binary_1 236
#define BIF_is_function_1 237
#define BIF_is_function_2 238
#define BIF_is_record_2 239
#define BIF_is_record_3 240
#define BIF_match_spec_test_3 241
#define BIF_ets_all_0 242
#define BIF_ets_new_2 243
#define BIF_ets_delete_1 244
#define BIF_ets_delete_2 245
#define BIF_ets_delete_all_objects_1 246
#define BIF_ets_delete_object_2 247
#define BIF_ets_first_1 248
#define BIF_ets_is_compiled_ms_1 249
#define BIF_ets_lookup_2 250
#define BIF_ets_lookup_element_3 251
#define BIF_ets_info_1 252
#define BIF_ets_info_2 253
#define BIF_ets_last_1 254
#define BIF_ets_match_1 255
#define BIF_ets_match_2 256
#define BIF_ets_match_3 257
#define BIF_ets_match_object_1 258
#define BIF_ets_match_object_2 259
#define BIF_ets_match_object_3 260
#define BIF_ets_member_2 261
#define BIF_ets_next_2 262
#define BIF_ets_prev_2 263
#define BIF_ets_insert_2 264
#define BIF_ets_insert_new_2 265
#define BIF_ets_rename_2 266
#define BIF_ets_safe_fixtable_2 267
#define BIF_ets_slot_2 268
#define BIF_ets_update_counter_3 269
#define BIF_ets_select_1 270
#define BIF_ets_select_2 271
#define BIF_ets_select_3 272
#define BIF_ets_select_count_2 273
#define BIF_ets_select_reverse_1 274
#define BIF_ets_select_reverse_2 275
#define BIF_ets_select_reverse_3 276
#define BIF_ets_select_delete_2 277
#define BIF_ets_match_spec_compile_1 278
#define BIF_ets_match_spec_run_r_3 279
#define BIF_os_putenv_2 280
#define BIF_os_getenv_0 281
#define BIF_os_getenv_1 282
#define BIF_os_getpid_0 283
#define BIF_os_timestamp_0 284
#define BIF_os_system_time_0 285
#define BIF_os_system_time_1 286
#define BIF_erl_ddll_try_load_3 287
#define BIF_erl_ddll_try_unload_2 288
#define BIF_erl_ddll_loaded_drivers_0 289
#define BIF_erl_ddll_info_2 290
#define BIF_erl_ddll_format_error_int_1 291
#define BIF_erl_ddll_monitor_2 292
#define BIF_erl_ddll_demonitor_1 293
#define BIF_re_compile_1 294
#define BIF_re_compile_2 295
#define BIF_re_run_2 296
#define BIF_re_run_3 297
#define BIF_lists_member_2 298
#define BIF_lists_reverse_2 299
#define BIF_lists_keymember_3 300
#define BIF_lists_keysearch_3 301
#define BIF_lists_keyfind_3 302
#define BIF_erts_debug_disassemble_1 303
#define BIF_erts_debug_breakpoint_2 304
#define BIF_erts_debug_same_2 305
#define BIF_erts_debug_flat_size_1 306
#define BIF_erts_debug_get_internal_state_1 307
#define BIF_erts_debug_set_internal_state_2 308
#define BIF_erts_debug_display_1 309
#define BIF_erts_debug_dist_ext_to_term_2 310
#define BIF_erts_debug_instructions_0 311
#define BIF_erts_debug_dump_monitors_1 312
#define BIF_erts_debug_dump_links_1 313
#define BIF_erts_debug_lock_counters_1 314
#define BIF_code_get_chunk_2 315
#define BIF_code_module_md5_1 316
#define BIF_code_make_stub_module_3 317
#define BIF_code_is_module_native_1 318
#define BIF_hibernate_3 319
#define BIF_error_logger_warning_map_0 320
#define BIF_get_module_info_1 321
#define BIF_get_module_info_2 322
#define BIF_is_boolean_1 323
#define BIF_string_to_integer_1 324
#define BIF_string_to_float_1 325
#define BIF_make_fun_3 326
#define BIF_iolist_size_1 327
#define BIF_iolist_to_binary_1 328
#define BIF_list_to_existing_atom_1 329
#define BIF_is_bitstring_1 330
#define BIF_tuple_size_1 331
#define BIF_byte_size_1 332
#define BIF_bit_size_1 333
#define BIF_list_to_bitstring_1 334
#define BIF_bitstring_to_list_1 335
#define BIF_ets_update_element_3 336
#define BIF_decode_packet_3 337
#define BIF_unicode_characters_to_binary_2 338
#define BIF_unicode_characters_to_list_2 339
#define BIF_unicode_bin_is_7bit_1 340
#define BIF_atom_to_binary_2 341
#define BIF_binary_to_atom_2 342
#define BIF_binary_to_existing_atom_2 343
#define BIF_net_kernel_dflag_unicode_io_1 344
#define BIF_ets_give_away_3 345
#define BIF_ets_setopts_2 346
#define BIF_load_nif_2 347
#define BIF_call_on_load_function_1 348
#define BIF_finish_after_on_load_2 349
#define BIF_binary_to_term_2 350
#define BIF_binary_part_2 351
#define BIF_binary_part_3 352
#define BIF_binary_compile_pattern_1 353
#define BIF_binary_match_2 354
#define BIF_binary_match_3 355
#define BIF_binary_matches_2 356
#define BIF_binary_matches_3 357
#define BIF_binary_longest_common_prefix_1 358
#define BIF_binary_longest_common_suffix_1 359
#define BIF_binary_first_1 360
#define BIF_binary_last_1 361
#define BIF_binary_at_2 362
#define BIF_binary_binary_part_2 363
#define BIF_binary_binary_part_3 364
#define BIF_binary_bin_to_list_1 365
#define BIF_binary_bin_to_list_2 366
#define BIF_binary_bin_to_list_3 367
#define BIF_binary_list_to_bin_1 368
#define BIF_binary_copy_1 369
#define BIF_binary_copy_2 370
#define BIF_binary_referenced_byte_size_1 371
#define BIF_binary_encode_unsigned_1 372
#define BIF_binary_encode_unsigned_2 373
#define BIF_binary_decode_unsigned_1 374
#define BIF_binary_decode_unsigned_2 375
#define BIF_nif_error_1 376
#define BIF_nif_error_2 377
#define BIF_prim_file_internal_name2native_1 378
#define BIF_prim_file_internal_native2name_1 379
#define BIF_prim_file_internal_normalize_utf8_1 380
#define BIF_prim_file_is_translatable_1 381
#define BIF_file_native_name_encoding_0 382
#define BIF_check_old_code_1 383
#define BIF_universaltime_to_posixtime_1 384
#define BIF_posixtime_to_universaltime_1 385
#define BIF_dt_put_tag_1 386
#define BIF_dt_get_tag_0 387
#define BIF_dt_get_tag_data_0 388
#define BIF_dt_spread_tag_1 389
#define BIF_dt_restore_tag_1 390
#define BIF_dt_prepend_vm_tag_data_1 391
#define BIF_dt_append_vm_tag_data_1 392
#define BIF_prepare_loading_2 393
#define BIF_finish_loading_1 394
#define BIF_insert_element_3 395
#define BIF_delete_element_2 396
#define BIF_binary_to_integer_1 397
#define BIF_binary_to_integer_2 398
#define BIF_integer_to_binary_1 399
#define BIF_list_to_integer_2 400
#define BIF_float_to_binary_1 401
#define BIF_float_to_binary_2 402
#define BIF_binary_to_float_1 403
#define BIF_io_printable_range_0 404
#define BIF_os_unsetenv_1 405
#define BIF_re_inspect_2 406
#define BIF_is_map_1 407
#define BIF_map_size_1 408
#define BIF_maps_to_list_1 409
#define BIF_maps_find_2 410
#define BIF_maps_get_2 411
#define BIF_maps_from_list_1 412
#define BIF_maps_is_key_2 413
#define BIF_maps_keys_1 414
#define BIF_maps_merge_2 415
#define BIF_maps_new_0 416
#define BIF_maps_put_3 417
#define BIF_maps_remove_2 418
#define BIF_maps_update_3 419
#define BIF_maps_values_1 420
#define BIF_erts_internal_cmp_term_2 421
#define BIF_ets_take_2 422
#define BIF_fun_info_mfa_1 423
#define BIF_get_keys_0 424
#define BIF_ets_update_counter_4 425
#define BIF_erts_debug_map_info_1 426
#define BIF_hash_2 427
#define BIF_hipe_bifs_write_u8_2 428
#define BIF_hipe_bifs_write_u32_2 429
#define BIF_hipe_bifs_bytearray_2 430
#define BIF_hipe_bifs_bytearray_sub_2 431
#define BIF_hipe_bifs_bytearray_update_3 432
#define BIF_hipe_bifs_bitarray_2 433
#define BIF_hipe_bifs_bitarray_sub_2 434
#define BIF_hipe_bifs_bitarray_update_3 435
#define BIF_hipe_bifs_array_2 436
#define BIF_hipe_bifs_array_length_1 437
#define BIF_hipe_bifs_array_sub_2 438
#define BIF_hipe_bifs_array_update_3 439
#define BIF_hipe_bifs_ref_1 440
#define BIF_hipe_bifs_ref_get_1 441
#define BIF_hipe_bifs_ref_set_2 442
#define BIF_hipe_bifs_enter_code_2 443
#define BIF_hipe_bifs_alloc_data_2 444
#define BIF_hipe_bifs_constants_size_0 445
#define BIF_hipe_bifs_merge_term_1 446
#define BIF_hipe_bifs_fun_to_address_1 447
#define BIF_hipe_bifs_set_native_address_3 448
#define BIF_hipe_bifs_set_funinfo_native_address_3 449
#define BIF_hipe_bifs_invalidate_funinfo_native_addresses_1 450
#define BIF_hipe_bifs_update_code_size_3 451
#define BIF_hipe_bifs_code_size_1 452
#define BIF_hipe_bifs_enter_sdesc_1 453
#define BIF_hipe_bifs_bif_address_3 454
#define BIF_hipe_bifs_primop_address_1 455
#define BIF_hipe_bifs_atom_to_word_1 456
#define BIF_hipe_bifs_term_to_word_1 457
#define BIF_hipe_bifs_get_fe_2 458
#define BIF_hipe_bifs_set_native_address_in_fe_2 459
#define BIF_hipe_bifs_find_na_or_make_stub_2 460
#define BIF_hipe_bifs_check_crc_1 461
#define BIF_hipe_bifs_system_crc_0 462
#define BIF_hipe_bifs_get_rts_param_1 463
#define BIF_hipe_bifs_patch_insn_3 464
#define BIF_hipe_bifs_patch_call_3 465
#define BIF_hipe_bifs_add_ref_2 466
#define BIF_hipe_bifs_mark_referred_from_1 467
#define BIF_hipe_bifs_remove_refs_from_1 468
#define BIF_hipe_bifs_redirect_referred_from_1 469
#define BIF_hipe_bifs_call_count_on_1 470
#define BIF_hipe_bifs_call_count_off_1 471
#define BIF_hipe_bifs_call_count_get_1 472
#define BIF_hipe_bifs_call_count_clear_1 473
#define BIF_hipe_bifs_trap_count_get_0 474
#define BIF_hipe_bifs_trap_count_clear_0 475
#define BIF_hipe_bifs_process_info_0 476
#define BIF_hipe_bifs_process_info_clear_0 477
#define BIF_hipe_bifs_message_info_0 478
#define BIF_hipe_bifs_message_info_clear_0 479
#define BIF_hipe_bifs_message_sizes_0 480
#define BIF_hipe_bifs_gc_info_0 481
#define BIF_hipe_bifs_shared_gc_info_0 482
#define BIF_hipe_bifs_incremental_gc_info_0 483
#define BIF_hipe_bifs_gc_info_clear_0 484
#define BIF_hipe_bifs_pause_times_0 485
#define BIF_hipe_bifs_system_timer_0 486
#define BIF_hipe_bifs_system_timer_clear_0 487
#define BIF_hipe_bifs_send_timer_0 488
#define BIF_hipe_bifs_send_timer_clear_0 489
#define BIF_hipe_bifs_gc_timer_0 490
#define BIF_hipe_bifs_shared_gc_timer_0 491
#define BIF_hipe_bifs_gc_timer_clear_0 492
#define BIF_hipe_bifs_misc_timer_0 493
#define BIF_hipe_bifs_misc_timer_clear_0 494
#define BIF_hipe_bifs_get_hrvtime_0 495
#define BIF_hipe_bifs_stop_hrvtime_0 496
#define BIF_hipe_bifs_show_estack_1 497
#define BIF_hipe_bifs_show_heap_1 498
#define BIF_hipe_bifs_show_nstack_1 499
#define BIF_hipe_bifs_nstack_used_size_0 500
#define BIF_hipe_bifs_show_pcb_1 501
#define BIF_hipe_bifs_show_term_1 502
#define BIF_hipe_bifs_in_native_0 503
#define BIF_hipe_bifs_modeswitch_debug_on_0 504
#define BIF_hipe_bifs_modeswitch_debug_off_0 505
#define BIF_hipe_bifs_debug_native_called_2 506
#define BIF_hipe_bifs_llvm_fix_pinned_regs_0 507

Eterm abs_1(Process*, Eterm*);
Eterm wrap_abs_1(Process*, Eterm*, UWord *I);
Eterm adler32_1(Process*, Eterm*);
Eterm wrap_adler32_1(Process*, Eterm*, UWord *I);
Eterm adler32_2(Process*, Eterm*);
Eterm wrap_adler32_2(Process*, Eterm*, UWord *I);
Eterm adler32_combine_3(Process*, Eterm*);
Eterm wrap_adler32_combine_3(Process*, Eterm*, UWord *I);
Eterm apply_3(Process*, Eterm*);
Eterm wrap_apply_3(Process*, Eterm*, UWord *I);
Eterm atom_to_list_1(Process*, Eterm*);
Eterm wrap_atom_to_list_1(Process*, Eterm*, UWord *I);
Eterm binary_to_list_1(Process*, Eterm*);
Eterm wrap_binary_to_list_1(Process*, Eterm*, UWord *I);
Eterm binary_to_list_3(Process*, Eterm*);
Eterm wrap_binary_to_list_3(Process*, Eterm*, UWord *I);
Eterm binary_to_term_1(Process*, Eterm*);
Eterm wrap_binary_to_term_1(Process*, Eterm*, UWord *I);
Eterm crc32_1(Process*, Eterm*);
Eterm wrap_crc32_1(Process*, Eterm*, UWord *I);
Eterm crc32_2(Process*, Eterm*);
Eterm wrap_crc32_2(Process*, Eterm*, UWord *I);
Eterm crc32_combine_3(Process*, Eterm*);
Eterm wrap_crc32_combine_3(Process*, Eterm*, UWord *I);
Eterm date_0(Process*, Eterm*);
Eterm wrap_date_0(Process*, Eterm*, UWord *I);
Eterm delete_module_1(Process*, Eterm*);
Eterm wrap_delete_module_1(Process*, Eterm*, UWord *I);
Eterm display_1(Process*, Eterm*);
Eterm wrap_display_1(Process*, Eterm*, UWord *I);
Eterm display_string_1(Process*, Eterm*);
Eterm wrap_display_string_1(Process*, Eterm*, UWord *I);
Eterm display_nl_0(Process*, Eterm*);
Eterm wrap_display_nl_0(Process*, Eterm*, UWord *I);
Eterm element_2(Process*, Eterm*);
Eterm wrap_element_2(Process*, Eterm*, UWord *I);
Eterm erase_0(Process*, Eterm*);
Eterm wrap_erase_0(Process*, Eterm*, UWord *I);
Eterm erase_1(Process*, Eterm*);
Eterm wrap_erase_1(Process*, Eterm*, UWord *I);
Eterm exit_1(Process*, Eterm*);
Eterm wrap_exit_1(Process*, Eterm*, UWord *I);
Eterm exit_2(Process*, Eterm*);
Eterm wrap_exit_2(Process*, Eterm*, UWord *I);
Eterm external_size_1(Process*, Eterm*);
Eterm wrap_external_size_1(Process*, Eterm*, UWord *I);
Eterm external_size_2(Process*, Eterm*);
Eterm wrap_external_size_2(Process*, Eterm*, UWord *I);
Eterm float_1(Process*, Eterm*);
Eterm wrap_float_1(Process*, Eterm*, UWord *I);
Eterm float_to_list_1(Process*, Eterm*);
Eterm wrap_float_to_list_1(Process*, Eterm*, UWord *I);
Eterm float_to_list_2(Process*, Eterm*);
Eterm wrap_float_to_list_2(Process*, Eterm*, UWord *I);
Eterm fun_info_2(Process*, Eterm*);
Eterm wrap_fun_info_2(Process*, Eterm*, UWord *I);
Eterm garbage_collect_0(Process*, Eterm*);
Eterm wrap_garbage_collect_0(Process*, Eterm*, UWord *I);
Eterm get_0(Process*, Eterm*);
Eterm wrap_get_0(Process*, Eterm*, UWord *I);
Eterm get_1(Process*, Eterm*);
Eterm wrap_get_1(Process*, Eterm*, UWord *I);
Eterm get_keys_1(Process*, Eterm*);
Eterm wrap_get_keys_1(Process*, Eterm*, UWord *I);
Eterm group_leader_0(Process*, Eterm*);
Eterm wrap_group_leader_0(Process*, Eterm*, UWord *I);
Eterm group_leader_2(Process*, Eterm*);
Eterm wrap_group_leader_2(Process*, Eterm*, UWord *I);
Eterm halt_0(Process*, Eterm*);
Eterm wrap_halt_0(Process*, Eterm*, UWord *I);
Eterm halt_1(Process*, Eterm*);
Eterm wrap_halt_1(Process*, Eterm*, UWord *I);
Eterm halt_2(Process*, Eterm*);
Eterm wrap_halt_2(Process*, Eterm*, UWord *I);
Eterm phash_2(Process*, Eterm*);
Eterm wrap_phash_2(Process*, Eterm*, UWord *I);
Eterm phash2_1(Process*, Eterm*);
Eterm wrap_phash2_1(Process*, Eterm*, UWord *I);
Eterm phash2_2(Process*, Eterm*);
Eterm wrap_phash2_2(Process*, Eterm*, UWord *I);
Eterm hd_1(Process*, Eterm*);
Eterm wrap_hd_1(Process*, Eterm*, UWord *I);
Eterm integer_to_list_1(Process*, Eterm*);
Eterm wrap_integer_to_list_1(Process*, Eterm*, UWord *I);
Eterm is_alive_0(Process*, Eterm*);
Eterm wrap_is_alive_0(Process*, Eterm*, UWord *I);
Eterm length_1(Process*, Eterm*);
Eterm wrap_length_1(Process*, Eterm*, UWord *I);
Eterm link_1(Process*, Eterm*);
Eterm wrap_link_1(Process*, Eterm*, UWord *I);
Eterm list_to_atom_1(Process*, Eterm*);
Eterm wrap_list_to_atom_1(Process*, Eterm*, UWord *I);
Eterm list_to_binary_1(Process*, Eterm*);
Eterm wrap_list_to_binary_1(Process*, Eterm*, UWord *I);
Eterm list_to_float_1(Process*, Eterm*);
Eterm wrap_list_to_float_1(Process*, Eterm*, UWord *I);
Eterm list_to_integer_1(Process*, Eterm*);
Eterm wrap_list_to_integer_1(Process*, Eterm*, UWord *I);
Eterm list_to_pid_1(Process*, Eterm*);
Eterm wrap_list_to_pid_1(Process*, Eterm*, UWord *I);
Eterm list_to_tuple_1(Process*, Eterm*);
Eterm wrap_list_to_tuple_1(Process*, Eterm*, UWord *I);
Eterm loaded_0(Process*, Eterm*);
Eterm wrap_loaded_0(Process*, Eterm*, UWord *I);
Eterm localtime_0(Process*, Eterm*);
Eterm wrap_localtime_0(Process*, Eterm*, UWord *I);
Eterm localtime_to_universaltime_2(Process*, Eterm*);
Eterm wrap_localtime_to_universaltime_2(Process*, Eterm*, UWord *I);
Eterm make_ref_0(Process*, Eterm*);
Eterm wrap_make_ref_0(Process*, Eterm*, UWord *I);
Eterm unique_integer_0(Process*, Eterm*);
Eterm wrap_unique_integer_0(Process*, Eterm*, UWord *I);
Eterm unique_integer_1(Process*, Eterm*);
Eterm wrap_unique_integer_1(Process*, Eterm*, UWord *I);
Eterm md5_1(Process*, Eterm*);
Eterm wrap_md5_1(Process*, Eterm*, UWord *I);
Eterm md5_init_0(Process*, Eterm*);
Eterm wrap_md5_init_0(Process*, Eterm*, UWord *I);
Eterm md5_update_2(Process*, Eterm*);
Eterm wrap_md5_update_2(Process*, Eterm*, UWord *I);
Eterm md5_final_1(Process*, Eterm*);
Eterm wrap_md5_final_1(Process*, Eterm*, UWord *I);
Eterm module_loaded_1(Process*, Eterm*);
Eterm wrap_module_loaded_1(Process*, Eterm*, UWord *I);
Eterm function_exported_3(Process*, Eterm*);
Eterm wrap_function_exported_3(Process*, Eterm*, UWord *I);
Eterm monitor_node_2(Process*, Eterm*);
Eterm wrap_monitor_node_2(Process*, Eterm*, UWord *I);
Eterm monitor_node_3(Process*, Eterm*);
Eterm wrap_monitor_node_3(Process*, Eterm*, UWord *I);
Eterm node_1(Process*, Eterm*);
Eterm wrap_node_1(Process*, Eterm*, UWord *I);
Eterm node_0(Process*, Eterm*);
Eterm wrap_node_0(Process*, Eterm*, UWord *I);
Eterm nodes_1(Process*, Eterm*);
Eterm wrap_nodes_1(Process*, Eterm*, UWord *I);
Eterm now_0(Process*, Eterm*);
Eterm wrap_now_0(Process*, Eterm*, UWord *I);
Eterm monotonic_time_0(Process*, Eterm*);
Eterm wrap_monotonic_time_0(Process*, Eterm*, UWord *I);
Eterm monotonic_time_1(Process*, Eterm*);
Eterm wrap_monotonic_time_1(Process*, Eterm*, UWord *I);
Eterm system_time_0(Process*, Eterm*);
Eterm wrap_system_time_0(Process*, Eterm*, UWord *I);
Eterm system_time_1(Process*, Eterm*);
Eterm wrap_system_time_1(Process*, Eterm*, UWord *I);
Eterm time_offset_0(Process*, Eterm*);
Eterm wrap_time_offset_0(Process*, Eterm*, UWord *I);
Eterm time_offset_1(Process*, Eterm*);
Eterm wrap_time_offset_1(Process*, Eterm*, UWord *I);
Eterm timestamp_0(Process*, Eterm*);
Eterm wrap_timestamp_0(Process*, Eterm*, UWord *I);
Eterm open_port_2(Process*, Eterm*);
Eterm wrap_open_port_2(Process*, Eterm*, UWord *I);
Eterm pid_to_list_1(Process*, Eterm*);
Eterm wrap_pid_to_list_1(Process*, Eterm*, UWord *I);
Eterm ports_0(Process*, Eterm*);
Eterm wrap_ports_0(Process*, Eterm*, UWord *I);
Eterm pre_loaded_0(Process*, Eterm*);
Eterm wrap_pre_loaded_0(Process*, Eterm*, UWord *I);
Eterm process_flag_2(Process*, Eterm*);
Eterm wrap_process_flag_2(Process*, Eterm*, UWord *I);
Eterm process_flag_3(Process*, Eterm*);
Eterm wrap_process_flag_3(Process*, Eterm*, UWord *I);
Eterm process_info_1(Process*, Eterm*);
Eterm wrap_process_info_1(Process*, Eterm*, UWord *I);
Eterm process_info_2(Process*, Eterm*);
Eterm wrap_process_info_2(Process*, Eterm*, UWord *I);
Eterm processes_0(Process*, Eterm*);
Eterm wrap_processes_0(Process*, Eterm*, UWord *I);
Eterm purge_module_1(Process*, Eterm*);
Eterm wrap_purge_module_1(Process*, Eterm*, UWord *I);
Eterm put_2(Process*, Eterm*);
Eterm wrap_put_2(Process*, Eterm*, UWord *I);
Eterm register_2(Process*, Eterm*);
Eterm wrap_register_2(Process*, Eterm*, UWord *I);
Eterm registered_0(Process*, Eterm*);
Eterm wrap_registered_0(Process*, Eterm*, UWord *I);
Eterm round_1(Process*, Eterm*);
Eterm wrap_round_1(Process*, Eterm*, UWord *I);
Eterm self_0(Process*, Eterm*);
Eterm wrap_self_0(Process*, Eterm*, UWord *I);
Eterm setelement_3(Process*, Eterm*);
Eterm wrap_setelement_3(Process*, Eterm*, UWord *I);
Eterm size_1(Process*, Eterm*);
Eterm wrap_size_1(Process*, Eterm*, UWord *I);
Eterm spawn_3(Process*, Eterm*);
Eterm wrap_spawn_3(Process*, Eterm*, UWord *I);
Eterm spawn_link_3(Process*, Eterm*);
Eterm wrap_spawn_link_3(Process*, Eterm*, UWord *I);
Eterm split_binary_2(Process*, Eterm*);
Eterm wrap_split_binary_2(Process*, Eterm*, UWord *I);
Eterm statistics_1(Process*, Eterm*);
Eterm wrap_statistics_1(Process*, Eterm*, UWord *I);
Eterm term_to_binary_1(Process*, Eterm*);
Eterm wrap_term_to_binary_1(Process*, Eterm*, UWord *I);
Eterm term_to_binary_2(Process*, Eterm*);
Eterm wrap_term_to_binary_2(Process*, Eterm*, UWord *I);
Eterm throw_1(Process*, Eterm*);
Eterm wrap_throw_1(Process*, Eterm*, UWord *I);
Eterm time_0(Process*, Eterm*);
Eterm wrap_time_0(Process*, Eterm*, UWord *I);
Eterm tl_1(Process*, Eterm*);
Eterm wrap_tl_1(Process*, Eterm*, UWord *I);
Eterm trunc_1(Process*, Eterm*);
Eterm wrap_trunc_1(Process*, Eterm*, UWord *I);
Eterm tuple_to_list_1(Process*, Eterm*);
Eterm wrap_tuple_to_list_1(Process*, Eterm*, UWord *I);
Eterm universaltime_0(Process*, Eterm*);
Eterm wrap_universaltime_0(Process*, Eterm*, UWord *I);
Eterm universaltime_to_localtime_1(Process*, Eterm*);
Eterm wrap_universaltime_to_localtime_1(Process*, Eterm*, UWord *I);
Eterm unlink_1(Process*, Eterm*);
Eterm wrap_unlink_1(Process*, Eterm*, UWord *I);
Eterm unregister_1(Process*, Eterm*);
Eterm wrap_unregister_1(Process*, Eterm*, UWord *I);
Eterm whereis_1(Process*, Eterm*);
Eterm wrap_whereis_1(Process*, Eterm*, UWord *I);
Eterm spawn_opt_1(Process*, Eterm*);
Eterm wrap_spawn_opt_1(Process*, Eterm*, UWord *I);
Eterm setnode_2(Process*, Eterm*);
Eterm wrap_setnode_2(Process*, Eterm*, UWord *I);
Eterm setnode_3(Process*, Eterm*);
Eterm wrap_setnode_3(Process*, Eterm*, UWord *I);
Eterm dist_exit_3(Process*, Eterm*);
Eterm wrap_dist_exit_3(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_info_1(Process*, Eterm*);
Eterm wrap_erts_internal_port_info_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_info_2(Process*, Eterm*);
Eterm wrap_erts_internal_port_info_2(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_call_3(Process*, Eterm*);
Eterm wrap_erts_internal_port_call_3(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_command_3(Process*, Eterm*);
Eterm wrap_erts_internal_port_command_3(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_control_3(Process*, Eterm*);
Eterm wrap_erts_internal_port_control_3(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_close_1(Process*, Eterm*);
Eterm wrap_erts_internal_port_close_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_port_connect_2(Process*, Eterm*);
Eterm wrap_erts_internal_port_connect_2(Process*, Eterm*, UWord *I);
Eterm erts_internal_request_system_task_3(Process*, Eterm*);
Eterm wrap_erts_internal_request_system_task_3(Process*, Eterm*, UWord *I);
Eterm erts_internal_check_process_code_2(Process*, Eterm*);
Eterm wrap_erts_internal_check_process_code_2(Process*, Eterm*, UWord *I);
Eterm erts_internal_map_to_tuple_keys_1(Process*, Eterm*);
Eterm wrap_erts_internal_map_to_tuple_keys_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_map_type_1(Process*, Eterm*);
Eterm wrap_erts_internal_map_type_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_map_hashmap_children_1(Process*, Eterm*);
Eterm wrap_erts_internal_map_hashmap_children_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_time_unit_0(Process*, Eterm*);
Eterm wrap_erts_internal_time_unit_0(Process*, Eterm*, UWord *I);
Eterm erts_internal_is_system_process_1(Process*, Eterm*);
Eterm wrap_erts_internal_is_system_process_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_system_check_1(Process*, Eterm*);
Eterm wrap_erts_internal_system_check_1(Process*, Eterm*, UWord *I);
Eterm port_set_data_2(Process*, Eterm*);
Eterm wrap_port_set_data_2(Process*, Eterm*, UWord *I);
Eterm port_get_data_1(Process*, Eterm*);
Eterm wrap_port_get_data_1(Process*, Eterm*, UWord *I);
Eterm trace_pattern_2(Process*, Eterm*);
Eterm wrap_trace_pattern_2(Process*, Eterm*, UWord *I);
Eterm trace_pattern_3(Process*, Eterm*);
Eterm wrap_trace_pattern_3(Process*, Eterm*, UWord *I);
Eterm trace_3(Process*, Eterm*);
Eterm wrap_trace_3(Process*, Eterm*, UWord *I);
Eterm trace_info_2(Process*, Eterm*);
Eterm wrap_trace_info_2(Process*, Eterm*, UWord *I);
Eterm trace_delivered_1(Process*, Eterm*);
Eterm wrap_trace_delivered_1(Process*, Eterm*, UWord *I);
Eterm seq_trace_2(Process*, Eterm*);
Eterm wrap_seq_trace_2(Process*, Eterm*, UWord *I);
Eterm seq_trace_info_1(Process*, Eterm*);
Eterm wrap_seq_trace_info_1(Process*, Eterm*, UWord *I);
Eterm seq_trace_print_1(Process*, Eterm*);
Eterm wrap_seq_trace_print_1(Process*, Eterm*, UWord *I);
Eterm seq_trace_print_2(Process*, Eterm*);
Eterm wrap_seq_trace_print_2(Process*, Eterm*, UWord *I);
Eterm suspend_process_2(Process*, Eterm*);
Eterm wrap_suspend_process_2(Process*, Eterm*, UWord *I);
Eterm resume_process_1(Process*, Eterm*);
Eterm wrap_resume_process_1(Process*, Eterm*, UWord *I);
Eterm process_display_2(Process*, Eterm*);
Eterm wrap_process_display_2(Process*, Eterm*, UWord *I);
Eterm bump_reductions_1(Process*, Eterm*);
Eterm wrap_bump_reductions_1(Process*, Eterm*, UWord *I);
Eterm math_cos_1(Process*, Eterm*);
Eterm wrap_math_cos_1(Process*, Eterm*, UWord *I);
Eterm math_cosh_1(Process*, Eterm*);
Eterm wrap_math_cosh_1(Process*, Eterm*, UWord *I);
Eterm math_sin_1(Process*, Eterm*);
Eterm wrap_math_sin_1(Process*, Eterm*, UWord *I);
Eterm math_sinh_1(Process*, Eterm*);
Eterm wrap_math_sinh_1(Process*, Eterm*, UWord *I);
Eterm math_tan_1(Process*, Eterm*);
Eterm wrap_math_tan_1(Process*, Eterm*, UWord *I);
Eterm math_tanh_1(Process*, Eterm*);
Eterm wrap_math_tanh_1(Process*, Eterm*, UWord *I);
Eterm math_acos_1(Process*, Eterm*);
Eterm wrap_math_acos_1(Process*, Eterm*, UWord *I);
Eterm math_acosh_1(Process*, Eterm*);
Eterm wrap_math_acosh_1(Process*, Eterm*, UWord *I);
Eterm math_asin_1(Process*, Eterm*);
Eterm wrap_math_asin_1(Process*, Eterm*, UWord *I);
Eterm math_asinh_1(Process*, Eterm*);
Eterm wrap_math_asinh_1(Process*, Eterm*, UWord *I);
Eterm math_atan_1(Process*, Eterm*);
Eterm wrap_math_atan_1(Process*, Eterm*, UWord *I);
Eterm math_atanh_1(Process*, Eterm*);
Eterm wrap_math_atanh_1(Process*, Eterm*, UWord *I);
Eterm math_erf_1(Process*, Eterm*);
Eterm wrap_math_erf_1(Process*, Eterm*, UWord *I);
Eterm math_erfc_1(Process*, Eterm*);
Eterm wrap_math_erfc_1(Process*, Eterm*, UWord *I);
Eterm math_exp_1(Process*, Eterm*);
Eterm wrap_math_exp_1(Process*, Eterm*, UWord *I);
Eterm math_log_1(Process*, Eterm*);
Eterm wrap_math_log_1(Process*, Eterm*, UWord *I);
Eterm math_log2_1(Process*, Eterm*);
Eterm wrap_math_log2_1(Process*, Eterm*, UWord *I);
Eterm math_log10_1(Process*, Eterm*);
Eterm wrap_math_log10_1(Process*, Eterm*, UWord *I);
Eterm math_sqrt_1(Process*, Eterm*);
Eterm wrap_math_sqrt_1(Process*, Eterm*, UWord *I);
Eterm math_atan2_2(Process*, Eterm*);
Eterm wrap_math_atan2_2(Process*, Eterm*, UWord *I);
Eterm math_pow_2(Process*, Eterm*);
Eterm wrap_math_pow_2(Process*, Eterm*, UWord *I);
Eterm start_timer_3(Process*, Eterm*);
Eterm wrap_start_timer_3(Process*, Eterm*, UWord *I);
Eterm start_timer_4(Process*, Eterm*);
Eterm wrap_start_timer_4(Process*, Eterm*, UWord *I);
Eterm send_after_3(Process*, Eterm*);
Eterm wrap_send_after_3(Process*, Eterm*, UWord *I);
Eterm send_after_4(Process*, Eterm*);
Eterm wrap_send_after_4(Process*, Eterm*, UWord *I);
Eterm cancel_timer_1(Process*, Eterm*);
Eterm wrap_cancel_timer_1(Process*, Eterm*, UWord *I);
Eterm cancel_timer_2(Process*, Eterm*);
Eterm wrap_cancel_timer_2(Process*, Eterm*, UWord *I);
Eterm read_timer_1(Process*, Eterm*);
Eterm wrap_read_timer_1(Process*, Eterm*, UWord *I);
Eterm read_timer_2(Process*, Eterm*);
Eterm wrap_read_timer_2(Process*, Eterm*, UWord *I);
Eterm make_tuple_2(Process*, Eterm*);
Eterm wrap_make_tuple_2(Process*, Eterm*, UWord *I);
Eterm append_element_2(Process*, Eterm*);
Eterm wrap_append_element_2(Process*, Eterm*, UWord *I);
Eterm make_tuple_3(Process*, Eterm*);
Eterm wrap_make_tuple_3(Process*, Eterm*, UWord *I);
Eterm system_flag_2(Process*, Eterm*);
Eterm wrap_system_flag_2(Process*, Eterm*, UWord *I);
Eterm system_info_1(Process*, Eterm*);
Eterm wrap_system_info_1(Process*, Eterm*, UWord *I);
Eterm system_monitor_0(Process*, Eterm*);
Eterm wrap_system_monitor_0(Process*, Eterm*, UWord *I);
Eterm system_monitor_1(Process*, Eterm*);
Eterm wrap_system_monitor_1(Process*, Eterm*, UWord *I);
Eterm system_monitor_2(Process*, Eterm*);
Eterm wrap_system_monitor_2(Process*, Eterm*, UWord *I);
Eterm system_profile_2(Process*, Eterm*);
Eterm wrap_system_profile_2(Process*, Eterm*, UWord *I);
Eterm system_profile_0(Process*, Eterm*);
Eterm wrap_system_profile_0(Process*, Eterm*, UWord *I);
Eterm ref_to_list_1(Process*, Eterm*);
Eterm wrap_ref_to_list_1(Process*, Eterm*, UWord *I);
Eterm port_to_list_1(Process*, Eterm*);
Eterm wrap_port_to_list_1(Process*, Eterm*, UWord *I);
Eterm fun_to_list_1(Process*, Eterm*);
Eterm wrap_fun_to_list_1(Process*, Eterm*, UWord *I);
Eterm monitor_2(Process*, Eterm*);
Eterm wrap_monitor_2(Process*, Eterm*, UWord *I);
Eterm demonitor_1(Process*, Eterm*);
Eterm wrap_demonitor_1(Process*, Eterm*, UWord *I);
Eterm demonitor_2(Process*, Eterm*);
Eterm wrap_demonitor_2(Process*, Eterm*, UWord *I);
Eterm is_process_alive_1(Process*, Eterm*);
Eterm wrap_is_process_alive_1(Process*, Eterm*, UWord *I);
Eterm error_1(Process*, Eterm*);
Eterm wrap_error_1(Process*, Eterm*, UWord *I);
Eterm error_2(Process*, Eterm*);
Eterm wrap_error_2(Process*, Eterm*, UWord *I);
Eterm raise_3(Process*, Eterm*);
Eterm wrap_raise_3(Process*, Eterm*, UWord *I);
Eterm get_stacktrace_0(Process*, Eterm*);
Eterm wrap_get_stacktrace_0(Process*, Eterm*, UWord *I);
Eterm is_builtin_3(Process*, Eterm*);
Eterm wrap_is_builtin_3(Process*, Eterm*, UWord *I);
Eterm and_2(Process*, Eterm*);
Eterm wrap_and_2(Process*, Eterm*, UWord *I);
Eterm or_2(Process*, Eterm*);
Eterm wrap_or_2(Process*, Eterm*, UWord *I);
Eterm xor_2(Process*, Eterm*);
Eterm wrap_xor_2(Process*, Eterm*, UWord *I);
Eterm not_1(Process*, Eterm*);
Eterm wrap_not_1(Process*, Eterm*, UWord *I);
Eterm sgt_2(Process*, Eterm*);
Eterm wrap_sgt_2(Process*, Eterm*, UWord *I);
Eterm sge_2(Process*, Eterm*);
Eterm wrap_sge_2(Process*, Eterm*, UWord *I);
Eterm slt_2(Process*, Eterm*);
Eterm wrap_slt_2(Process*, Eterm*, UWord *I);
Eterm sle_2(Process*, Eterm*);
Eterm wrap_sle_2(Process*, Eterm*, UWord *I);
Eterm seq_2(Process*, Eterm*);
Eterm wrap_seq_2(Process*, Eterm*, UWord *I);
Eterm seqeq_2(Process*, Eterm*);
Eterm wrap_seqeq_2(Process*, Eterm*, UWord *I);
Eterm sneq_2(Process*, Eterm*);
Eterm wrap_sneq_2(Process*, Eterm*, UWord *I);
Eterm sneqeq_2(Process*, Eterm*);
Eterm wrap_sneqeq_2(Process*, Eterm*, UWord *I);
Eterm splus_2(Process*, Eterm*);
Eterm wrap_splus_2(Process*, Eterm*, UWord *I);
Eterm sminus_2(Process*, Eterm*);
Eterm wrap_sminus_2(Process*, Eterm*, UWord *I);
Eterm stimes_2(Process*, Eterm*);
Eterm wrap_stimes_2(Process*, Eterm*, UWord *I);
Eterm div_2(Process*, Eterm*);
Eterm wrap_div_2(Process*, Eterm*, UWord *I);
Eterm intdiv_2(Process*, Eterm*);
Eterm wrap_intdiv_2(Process*, Eterm*, UWord *I);
Eterm rem_2(Process*, Eterm*);
Eterm wrap_rem_2(Process*, Eterm*, UWord *I);
Eterm bor_2(Process*, Eterm*);
Eterm wrap_bor_2(Process*, Eterm*, UWord *I);
Eterm band_2(Process*, Eterm*);
Eterm wrap_band_2(Process*, Eterm*, UWord *I);
Eterm bxor_2(Process*, Eterm*);
Eterm wrap_bxor_2(Process*, Eterm*, UWord *I);
Eterm bsl_2(Process*, Eterm*);
Eterm wrap_bsl_2(Process*, Eterm*, UWord *I);
Eterm bsr_2(Process*, Eterm*);
Eterm wrap_bsr_2(Process*, Eterm*, UWord *I);
Eterm bnot_1(Process*, Eterm*);
Eterm wrap_bnot_1(Process*, Eterm*, UWord *I);
Eterm sminus_1(Process*, Eterm*);
Eterm wrap_sminus_1(Process*, Eterm*, UWord *I);
Eterm splus_1(Process*, Eterm*);
Eterm wrap_splus_1(Process*, Eterm*, UWord *I);
Eterm ebif_bang_2(Process*, Eterm*);
Eterm wrap_ebif_bang_2(Process*, Eterm*, UWord *I);
Eterm send_2(Process*, Eterm*);
Eterm wrap_send_2(Process*, Eterm*, UWord *I);
Eterm send_3(Process*, Eterm*);
Eterm wrap_send_3(Process*, Eterm*, UWord *I);
Eterm ebif_plusplus_2(Process*, Eterm*);
Eterm wrap_ebif_plusplus_2(Process*, Eterm*, UWord *I);
Eterm append_2(Process*, Eterm*);
Eterm wrap_append_2(Process*, Eterm*, UWord *I);
Eterm ebif_minusminus_2(Process*, Eterm*);
Eterm wrap_ebif_minusminus_2(Process*, Eterm*, UWord *I);
Eterm subtract_2(Process*, Eterm*);
Eterm wrap_subtract_2(Process*, Eterm*, UWord *I);
Eterm is_atom_1(Process*, Eterm*);
Eterm wrap_is_atom_1(Process*, Eterm*, UWord *I);
Eterm is_list_1(Process*, Eterm*);
Eterm wrap_is_list_1(Process*, Eterm*, UWord *I);
Eterm is_tuple_1(Process*, Eterm*);
Eterm wrap_is_tuple_1(Process*, Eterm*, UWord *I);
Eterm is_float_1(Process*, Eterm*);
Eterm wrap_is_float_1(Process*, Eterm*, UWord *I);
Eterm is_integer_1(Process*, Eterm*);
Eterm wrap_is_integer_1(Process*, Eterm*, UWord *I);
Eterm is_number_1(Process*, Eterm*);
Eterm wrap_is_number_1(Process*, Eterm*, UWord *I);
Eterm is_pid_1(Process*, Eterm*);
Eterm wrap_is_pid_1(Process*, Eterm*, UWord *I);
Eterm is_port_1(Process*, Eterm*);
Eterm wrap_is_port_1(Process*, Eterm*, UWord *I);
Eterm is_reference_1(Process*, Eterm*);
Eterm wrap_is_reference_1(Process*, Eterm*, UWord *I);
Eterm is_binary_1(Process*, Eterm*);
Eterm wrap_is_binary_1(Process*, Eterm*, UWord *I);
Eterm is_function_1(Process*, Eterm*);
Eterm wrap_is_function_1(Process*, Eterm*, UWord *I);
Eterm is_function_2(Process*, Eterm*);
Eterm wrap_is_function_2(Process*, Eterm*, UWord *I);
Eterm is_record_2(Process*, Eterm*);
Eterm wrap_is_record_2(Process*, Eterm*, UWord *I);
Eterm is_record_3(Process*, Eterm*);
Eterm wrap_is_record_3(Process*, Eterm*, UWord *I);
Eterm match_spec_test_3(Process*, Eterm*);
Eterm wrap_match_spec_test_3(Process*, Eterm*, UWord *I);
Eterm ets_all_0(Process*, Eterm*);
Eterm wrap_ets_all_0(Process*, Eterm*, UWord *I);
Eterm ets_new_2(Process*, Eterm*);
Eterm wrap_ets_new_2(Process*, Eterm*, UWord *I);
Eterm ets_delete_1(Process*, Eterm*);
Eterm wrap_ets_delete_1(Process*, Eterm*, UWord *I);
Eterm ets_delete_2(Process*, Eterm*);
Eterm wrap_ets_delete_2(Process*, Eterm*, UWord *I);
Eterm ets_delete_all_objects_1(Process*, Eterm*);
Eterm wrap_ets_delete_all_objects_1(Process*, Eterm*, UWord *I);
Eterm ets_delete_object_2(Process*, Eterm*);
Eterm wrap_ets_delete_object_2(Process*, Eterm*, UWord *I);
Eterm ets_first_1(Process*, Eterm*);
Eterm wrap_ets_first_1(Process*, Eterm*, UWord *I);
Eterm ets_is_compiled_ms_1(Process*, Eterm*);
Eterm wrap_ets_is_compiled_ms_1(Process*, Eterm*, UWord *I);
Eterm ets_lookup_2(Process*, Eterm*);
Eterm wrap_ets_lookup_2(Process*, Eterm*, UWord *I);
Eterm ets_lookup_element_3(Process*, Eterm*);
Eterm wrap_ets_lookup_element_3(Process*, Eterm*, UWord *I);
Eterm ets_info_1(Process*, Eterm*);
Eterm wrap_ets_info_1(Process*, Eterm*, UWord *I);
Eterm ets_info_2(Process*, Eterm*);
Eterm wrap_ets_info_2(Process*, Eterm*, UWord *I);
Eterm ets_last_1(Process*, Eterm*);
Eterm wrap_ets_last_1(Process*, Eterm*, UWord *I);
Eterm ets_match_1(Process*, Eterm*);
Eterm wrap_ets_match_1(Process*, Eterm*, UWord *I);
Eterm ets_match_2(Process*, Eterm*);
Eterm wrap_ets_match_2(Process*, Eterm*, UWord *I);
Eterm ets_match_3(Process*, Eterm*);
Eterm wrap_ets_match_3(Process*, Eterm*, UWord *I);
Eterm ets_match_object_1(Process*, Eterm*);
Eterm wrap_ets_match_object_1(Process*, Eterm*, UWord *I);
Eterm ets_match_object_2(Process*, Eterm*);
Eterm wrap_ets_match_object_2(Process*, Eterm*, UWord *I);
Eterm ets_match_object_3(Process*, Eterm*);
Eterm wrap_ets_match_object_3(Process*, Eterm*, UWord *I);
Eterm ets_member_2(Process*, Eterm*);
Eterm wrap_ets_member_2(Process*, Eterm*, UWord *I);
Eterm ets_next_2(Process*, Eterm*);
Eterm wrap_ets_next_2(Process*, Eterm*, UWord *I);
Eterm ets_prev_2(Process*, Eterm*);
Eterm wrap_ets_prev_2(Process*, Eterm*, UWord *I);
Eterm ets_insert_2(Process*, Eterm*);
Eterm wrap_ets_insert_2(Process*, Eterm*, UWord *I);
Eterm ets_insert_new_2(Process*, Eterm*);
Eterm wrap_ets_insert_new_2(Process*, Eterm*, UWord *I);
Eterm ets_rename_2(Process*, Eterm*);
Eterm wrap_ets_rename_2(Process*, Eterm*, UWord *I);
Eterm ets_safe_fixtable_2(Process*, Eterm*);
Eterm wrap_ets_safe_fixtable_2(Process*, Eterm*, UWord *I);
Eterm ets_slot_2(Process*, Eterm*);
Eterm wrap_ets_slot_2(Process*, Eterm*, UWord *I);
Eterm ets_update_counter_3(Process*, Eterm*);
Eterm wrap_ets_update_counter_3(Process*, Eterm*, UWord *I);
Eterm ets_select_1(Process*, Eterm*);
Eterm wrap_ets_select_1(Process*, Eterm*, UWord *I);
Eterm ets_select_2(Process*, Eterm*);
Eterm wrap_ets_select_2(Process*, Eterm*, UWord *I);
Eterm ets_select_3(Process*, Eterm*);
Eterm wrap_ets_select_3(Process*, Eterm*, UWord *I);
Eterm ets_select_count_2(Process*, Eterm*);
Eterm wrap_ets_select_count_2(Process*, Eterm*, UWord *I);
Eterm ets_select_reverse_1(Process*, Eterm*);
Eterm wrap_ets_select_reverse_1(Process*, Eterm*, UWord *I);
Eterm ets_select_reverse_2(Process*, Eterm*);
Eterm wrap_ets_select_reverse_2(Process*, Eterm*, UWord *I);
Eterm ets_select_reverse_3(Process*, Eterm*);
Eterm wrap_ets_select_reverse_3(Process*, Eterm*, UWord *I);
Eterm ets_select_delete_2(Process*, Eterm*);
Eterm wrap_ets_select_delete_2(Process*, Eterm*, UWord *I);
Eterm ets_match_spec_compile_1(Process*, Eterm*);
Eterm wrap_ets_match_spec_compile_1(Process*, Eterm*, UWord *I);
Eterm ets_match_spec_run_r_3(Process*, Eterm*);
Eterm wrap_ets_match_spec_run_r_3(Process*, Eterm*, UWord *I);
Eterm os_putenv_2(Process*, Eterm*);
Eterm wrap_os_putenv_2(Process*, Eterm*, UWord *I);
Eterm os_getenv_0(Process*, Eterm*);
Eterm wrap_os_getenv_0(Process*, Eterm*, UWord *I);
Eterm os_getenv_1(Process*, Eterm*);
Eterm wrap_os_getenv_1(Process*, Eterm*, UWord *I);
Eterm os_getpid_0(Process*, Eterm*);
Eterm wrap_os_getpid_0(Process*, Eterm*, UWord *I);
Eterm os_timestamp_0(Process*, Eterm*);
Eterm wrap_os_timestamp_0(Process*, Eterm*, UWord *I);
Eterm os_system_time_0(Process*, Eterm*);
Eterm wrap_os_system_time_0(Process*, Eterm*, UWord *I);
Eterm os_system_time_1(Process*, Eterm*);
Eterm wrap_os_system_time_1(Process*, Eterm*, UWord *I);
Eterm erl_ddll_try_load_3(Process*, Eterm*);
Eterm wrap_erl_ddll_try_load_3(Process*, Eterm*, UWord *I);
Eterm erl_ddll_try_unload_2(Process*, Eterm*);
Eterm wrap_erl_ddll_try_unload_2(Process*, Eterm*, UWord *I);
Eterm erl_ddll_loaded_drivers_0(Process*, Eterm*);
Eterm wrap_erl_ddll_loaded_drivers_0(Process*, Eterm*, UWord *I);
Eterm erl_ddll_info_2(Process*, Eterm*);
Eterm wrap_erl_ddll_info_2(Process*, Eterm*, UWord *I);
Eterm erl_ddll_format_error_int_1(Process*, Eterm*);
Eterm wrap_erl_ddll_format_error_int_1(Process*, Eterm*, UWord *I);
Eterm erl_ddll_monitor_2(Process*, Eterm*);
Eterm wrap_erl_ddll_monitor_2(Process*, Eterm*, UWord *I);
Eterm erl_ddll_demonitor_1(Process*, Eterm*);
Eterm wrap_erl_ddll_demonitor_1(Process*, Eterm*, UWord *I);
Eterm re_compile_1(Process*, Eterm*);
Eterm wrap_re_compile_1(Process*, Eterm*, UWord *I);
Eterm re_compile_2(Process*, Eterm*);
Eterm wrap_re_compile_2(Process*, Eterm*, UWord *I);
Eterm re_run_2(Process*, Eterm*);
Eterm wrap_re_run_2(Process*, Eterm*, UWord *I);
Eterm re_run_3(Process*, Eterm*);
Eterm wrap_re_run_3(Process*, Eterm*, UWord *I);
Eterm lists_member_2(Process*, Eterm*);
Eterm wrap_lists_member_2(Process*, Eterm*, UWord *I);
Eterm lists_reverse_2(Process*, Eterm*);
Eterm wrap_lists_reverse_2(Process*, Eterm*, UWord *I);
Eterm lists_keymember_3(Process*, Eterm*);
Eterm wrap_lists_keymember_3(Process*, Eterm*, UWord *I);
Eterm lists_keysearch_3(Process*, Eterm*);
Eterm wrap_lists_keysearch_3(Process*, Eterm*, UWord *I);
Eterm lists_keyfind_3(Process*, Eterm*);
Eterm wrap_lists_keyfind_3(Process*, Eterm*, UWord *I);
Eterm erts_debug_disassemble_1(Process*, Eterm*);
Eterm wrap_erts_debug_disassemble_1(Process*, Eterm*, UWord *I);
Eterm erts_debug_breakpoint_2(Process*, Eterm*);
Eterm wrap_erts_debug_breakpoint_2(Process*, Eterm*, UWord *I);
Eterm erts_debug_same_2(Process*, Eterm*);
Eterm wrap_erts_debug_same_2(Process*, Eterm*, UWord *I);
Eterm erts_debug_flat_size_1(Process*, Eterm*);
Eterm wrap_erts_debug_flat_size_1(Process*, Eterm*, UWord *I);
Eterm erts_debug_get_internal_state_1(Process*, Eterm*);
Eterm wrap_erts_debug_get_internal_state_1(Process*, Eterm*, UWord *I);
Eterm erts_debug_set_internal_state_2(Process*, Eterm*);
Eterm wrap_erts_debug_set_internal_state_2(Process*, Eterm*, UWord *I);
Eterm erts_debug_display_1(Process*, Eterm*);
Eterm wrap_erts_debug_display_1(Process*, Eterm*, UWord *I);
Eterm erts_debug_dist_ext_to_term_2(Process*, Eterm*);
Eterm wrap_erts_debug_dist_ext_to_term_2(Process*, Eterm*, UWord *I);
Eterm erts_debug_instructions_0(Process*, Eterm*);
Eterm wrap_erts_debug_instructions_0(Process*, Eterm*, UWord *I);
Eterm erts_debug_dump_monitors_1(Process*, Eterm*);
Eterm wrap_erts_debug_dump_monitors_1(Process*, Eterm*, UWord *I);
Eterm erts_debug_dump_links_1(Process*, Eterm*);
Eterm wrap_erts_debug_dump_links_1(Process*, Eterm*, UWord *I);
Eterm erts_debug_lock_counters_1(Process*, Eterm*);
Eterm wrap_erts_debug_lock_counters_1(Process*, Eterm*, UWord *I);
Eterm code_get_chunk_2(Process*, Eterm*);
Eterm wrap_code_get_chunk_2(Process*, Eterm*, UWord *I);
Eterm code_module_md5_1(Process*, Eterm*);
Eterm wrap_code_module_md5_1(Process*, Eterm*, UWord *I);
Eterm code_make_stub_module_3(Process*, Eterm*);
Eterm wrap_code_make_stub_module_3(Process*, Eterm*, UWord *I);
Eterm code_is_module_native_1(Process*, Eterm*);
Eterm wrap_code_is_module_native_1(Process*, Eterm*, UWord *I);
Eterm hibernate_3(Process*, Eterm*);
Eterm wrap_hibernate_3(Process*, Eterm*, UWord *I);
Eterm error_logger_warning_map_0(Process*, Eterm*);
Eterm wrap_error_logger_warning_map_0(Process*, Eterm*, UWord *I);
Eterm get_module_info_1(Process*, Eterm*);
Eterm wrap_get_module_info_1(Process*, Eterm*, UWord *I);
Eterm get_module_info_2(Process*, Eterm*);
Eterm wrap_get_module_info_2(Process*, Eterm*, UWord *I);
Eterm is_boolean_1(Process*, Eterm*);
Eterm wrap_is_boolean_1(Process*, Eterm*, UWord *I);
Eterm string_to_integer_1(Process*, Eterm*);
Eterm wrap_string_to_integer_1(Process*, Eterm*, UWord *I);
Eterm string_to_float_1(Process*, Eterm*);
Eterm wrap_string_to_float_1(Process*, Eterm*, UWord *I);
Eterm make_fun_3(Process*, Eterm*);
Eterm wrap_make_fun_3(Process*, Eterm*, UWord *I);
Eterm iolist_size_1(Process*, Eterm*);
Eterm wrap_iolist_size_1(Process*, Eterm*, UWord *I);
Eterm iolist_to_binary_1(Process*, Eterm*);
Eterm wrap_iolist_to_binary_1(Process*, Eterm*, UWord *I);
Eterm list_to_existing_atom_1(Process*, Eterm*);
Eterm wrap_list_to_existing_atom_1(Process*, Eterm*, UWord *I);
Eterm is_bitstring_1(Process*, Eterm*);
Eterm wrap_is_bitstring_1(Process*, Eterm*, UWord *I);
Eterm tuple_size_1(Process*, Eterm*);
Eterm wrap_tuple_size_1(Process*, Eterm*, UWord *I);
Eterm byte_size_1(Process*, Eterm*);
Eterm wrap_byte_size_1(Process*, Eterm*, UWord *I);
Eterm bit_size_1(Process*, Eterm*);
Eterm wrap_bit_size_1(Process*, Eterm*, UWord *I);
Eterm list_to_bitstring_1(Process*, Eterm*);
Eterm wrap_list_to_bitstring_1(Process*, Eterm*, UWord *I);
Eterm bitstring_to_list_1(Process*, Eterm*);
Eterm wrap_bitstring_to_list_1(Process*, Eterm*, UWord *I);
Eterm ets_update_element_3(Process*, Eterm*);
Eterm wrap_ets_update_element_3(Process*, Eterm*, UWord *I);
Eterm decode_packet_3(Process*, Eterm*);
Eterm wrap_decode_packet_3(Process*, Eterm*, UWord *I);
Eterm unicode_characters_to_binary_2(Process*, Eterm*);
Eterm wrap_unicode_characters_to_binary_2(Process*, Eterm*, UWord *I);
Eterm unicode_characters_to_list_2(Process*, Eterm*);
Eterm wrap_unicode_characters_to_list_2(Process*, Eterm*, UWord *I);
Eterm unicode_bin_is_7bit_1(Process*, Eterm*);
Eterm wrap_unicode_bin_is_7bit_1(Process*, Eterm*, UWord *I);
Eterm atom_to_binary_2(Process*, Eterm*);
Eterm wrap_atom_to_binary_2(Process*, Eterm*, UWord *I);
Eterm binary_to_atom_2(Process*, Eterm*);
Eterm wrap_binary_to_atom_2(Process*, Eterm*, UWord *I);
Eterm binary_to_existing_atom_2(Process*, Eterm*);
Eterm wrap_binary_to_existing_atom_2(Process*, Eterm*, UWord *I);
Eterm net_kernel_dflag_unicode_io_1(Process*, Eterm*);
Eterm wrap_net_kernel_dflag_unicode_io_1(Process*, Eterm*, UWord *I);
Eterm ets_give_away_3(Process*, Eterm*);
Eterm wrap_ets_give_away_3(Process*, Eterm*, UWord *I);
Eterm ets_setopts_2(Process*, Eterm*);
Eterm wrap_ets_setopts_2(Process*, Eterm*, UWord *I);
Eterm load_nif_2(Process*, Eterm*);
Eterm wrap_load_nif_2(Process*, Eterm*, UWord *I);
Eterm call_on_load_function_1(Process*, Eterm*);
Eterm wrap_call_on_load_function_1(Process*, Eterm*, UWord *I);
Eterm finish_after_on_load_2(Process*, Eterm*);
Eterm wrap_finish_after_on_load_2(Process*, Eterm*, UWord *I);
Eterm binary_to_term_2(Process*, Eterm*);
Eterm wrap_binary_to_term_2(Process*, Eterm*, UWord *I);
Eterm binary_part_2(Process*, Eterm*);
Eterm wrap_binary_part_2(Process*, Eterm*, UWord *I);
Eterm binary_part_3(Process*, Eterm*);
Eterm wrap_binary_part_3(Process*, Eterm*, UWord *I);
Eterm binary_compile_pattern_1(Process*, Eterm*);
Eterm wrap_binary_compile_pattern_1(Process*, Eterm*, UWord *I);
Eterm binary_match_2(Process*, Eterm*);
Eterm wrap_binary_match_2(Process*, Eterm*, UWord *I);
Eterm binary_match_3(Process*, Eterm*);
Eterm wrap_binary_match_3(Process*, Eterm*, UWord *I);
Eterm binary_matches_2(Process*, Eterm*);
Eterm wrap_binary_matches_2(Process*, Eterm*, UWord *I);
Eterm binary_matches_3(Process*, Eterm*);
Eterm wrap_binary_matches_3(Process*, Eterm*, UWord *I);
Eterm binary_longest_common_prefix_1(Process*, Eterm*);
Eterm wrap_binary_longest_common_prefix_1(Process*, Eterm*, UWord *I);
Eterm binary_longest_common_suffix_1(Process*, Eterm*);
Eterm wrap_binary_longest_common_suffix_1(Process*, Eterm*, UWord *I);
Eterm binary_first_1(Process*, Eterm*);
Eterm wrap_binary_first_1(Process*, Eterm*, UWord *I);
Eterm binary_last_1(Process*, Eterm*);
Eterm wrap_binary_last_1(Process*, Eterm*, UWord *I);
Eterm binary_at_2(Process*, Eterm*);
Eterm wrap_binary_at_2(Process*, Eterm*, UWord *I);
Eterm binary_binary_part_2(Process*, Eterm*);
Eterm wrap_binary_binary_part_2(Process*, Eterm*, UWord *I);
Eterm binary_binary_part_3(Process*, Eterm*);
Eterm wrap_binary_binary_part_3(Process*, Eterm*, UWord *I);
Eterm binary_bin_to_list_1(Process*, Eterm*);
Eterm wrap_binary_bin_to_list_1(Process*, Eterm*, UWord *I);
Eterm binary_bin_to_list_2(Process*, Eterm*);
Eterm wrap_binary_bin_to_list_2(Process*, Eterm*, UWord *I);
Eterm binary_bin_to_list_3(Process*, Eterm*);
Eterm wrap_binary_bin_to_list_3(Process*, Eterm*, UWord *I);
Eterm binary_list_to_bin_1(Process*, Eterm*);
Eterm wrap_binary_list_to_bin_1(Process*, Eterm*, UWord *I);
Eterm binary_copy_1(Process*, Eterm*);
Eterm wrap_binary_copy_1(Process*, Eterm*, UWord *I);
Eterm binary_copy_2(Process*, Eterm*);
Eterm wrap_binary_copy_2(Process*, Eterm*, UWord *I);
Eterm binary_referenced_byte_size_1(Process*, Eterm*);
Eterm wrap_binary_referenced_byte_size_1(Process*, Eterm*, UWord *I);
Eterm binary_encode_unsigned_1(Process*, Eterm*);
Eterm wrap_binary_encode_unsigned_1(Process*, Eterm*, UWord *I);
Eterm binary_encode_unsigned_2(Process*, Eterm*);
Eterm wrap_binary_encode_unsigned_2(Process*, Eterm*, UWord *I);
Eterm binary_decode_unsigned_1(Process*, Eterm*);
Eterm wrap_binary_decode_unsigned_1(Process*, Eterm*, UWord *I);
Eterm binary_decode_unsigned_2(Process*, Eterm*);
Eterm wrap_binary_decode_unsigned_2(Process*, Eterm*, UWord *I);
Eterm nif_error_1(Process*, Eterm*);
Eterm wrap_nif_error_1(Process*, Eterm*, UWord *I);
Eterm nif_error_2(Process*, Eterm*);
Eterm wrap_nif_error_2(Process*, Eterm*, UWord *I);
Eterm prim_file_internal_name2native_1(Process*, Eterm*);
Eterm wrap_prim_file_internal_name2native_1(Process*, Eterm*, UWord *I);
Eterm prim_file_internal_native2name_1(Process*, Eterm*);
Eterm wrap_prim_file_internal_native2name_1(Process*, Eterm*, UWord *I);
Eterm prim_file_internal_normalize_utf8_1(Process*, Eterm*);
Eterm wrap_prim_file_internal_normalize_utf8_1(Process*, Eterm*, UWord *I);
Eterm prim_file_is_translatable_1(Process*, Eterm*);
Eterm wrap_prim_file_is_translatable_1(Process*, Eterm*, UWord *I);
Eterm file_native_name_encoding_0(Process*, Eterm*);
Eterm wrap_file_native_name_encoding_0(Process*, Eterm*, UWord *I);
Eterm check_old_code_1(Process*, Eterm*);
Eterm wrap_check_old_code_1(Process*, Eterm*, UWord *I);
Eterm universaltime_to_posixtime_1(Process*, Eterm*);
Eterm wrap_universaltime_to_posixtime_1(Process*, Eterm*, UWord *I);
Eterm posixtime_to_universaltime_1(Process*, Eterm*);
Eterm wrap_posixtime_to_universaltime_1(Process*, Eterm*, UWord *I);
Eterm dt_put_tag_1(Process*, Eterm*);
Eterm wrap_dt_put_tag_1(Process*, Eterm*, UWord *I);
Eterm dt_get_tag_0(Process*, Eterm*);
Eterm wrap_dt_get_tag_0(Process*, Eterm*, UWord *I);
Eterm dt_get_tag_data_0(Process*, Eterm*);
Eterm wrap_dt_get_tag_data_0(Process*, Eterm*, UWord *I);
Eterm dt_spread_tag_1(Process*, Eterm*);
Eterm wrap_dt_spread_tag_1(Process*, Eterm*, UWord *I);
Eterm dt_restore_tag_1(Process*, Eterm*);
Eterm wrap_dt_restore_tag_1(Process*, Eterm*, UWord *I);
Eterm dt_prepend_vm_tag_data_1(Process*, Eterm*);
Eterm wrap_dt_prepend_vm_tag_data_1(Process*, Eterm*, UWord *I);
Eterm dt_append_vm_tag_data_1(Process*, Eterm*);
Eterm wrap_dt_append_vm_tag_data_1(Process*, Eterm*, UWord *I);
Eterm prepare_loading_2(Process*, Eterm*);
Eterm wrap_prepare_loading_2(Process*, Eterm*, UWord *I);
Eterm finish_loading_1(Process*, Eterm*);
Eterm wrap_finish_loading_1(Process*, Eterm*, UWord *I);
Eterm insert_element_3(Process*, Eterm*);
Eterm wrap_insert_element_3(Process*, Eterm*, UWord *I);
Eterm delete_element_2(Process*, Eterm*);
Eterm wrap_delete_element_2(Process*, Eterm*, UWord *I);
Eterm binary_to_integer_1(Process*, Eterm*);
Eterm wrap_binary_to_integer_1(Process*, Eterm*, UWord *I);
Eterm binary_to_integer_2(Process*, Eterm*);
Eterm wrap_binary_to_integer_2(Process*, Eterm*, UWord *I);
Eterm integer_to_binary_1(Process*, Eterm*);
Eterm wrap_integer_to_binary_1(Process*, Eterm*, UWord *I);
Eterm list_to_integer_2(Process*, Eterm*);
Eterm wrap_list_to_integer_2(Process*, Eterm*, UWord *I);
Eterm float_to_binary_1(Process*, Eterm*);
Eterm wrap_float_to_binary_1(Process*, Eterm*, UWord *I);
Eterm float_to_binary_2(Process*, Eterm*);
Eterm wrap_float_to_binary_2(Process*, Eterm*, UWord *I);
Eterm binary_to_float_1(Process*, Eterm*);
Eterm wrap_binary_to_float_1(Process*, Eterm*, UWord *I);
Eterm io_printable_range_0(Process*, Eterm*);
Eterm wrap_io_printable_range_0(Process*, Eterm*, UWord *I);
Eterm os_unsetenv_1(Process*, Eterm*);
Eterm wrap_os_unsetenv_1(Process*, Eterm*, UWord *I);
Eterm re_inspect_2(Process*, Eterm*);
Eterm wrap_re_inspect_2(Process*, Eterm*, UWord *I);
Eterm is_map_1(Process*, Eterm*);
Eterm wrap_is_map_1(Process*, Eterm*, UWord *I);
Eterm map_size_1(Process*, Eterm*);
Eterm wrap_map_size_1(Process*, Eterm*, UWord *I);
Eterm maps_to_list_1(Process*, Eterm*);
Eterm wrap_maps_to_list_1(Process*, Eterm*, UWord *I);
Eterm maps_find_2(Process*, Eterm*);
Eterm wrap_maps_find_2(Process*, Eterm*, UWord *I);
Eterm maps_get_2(Process*, Eterm*);
Eterm wrap_maps_get_2(Process*, Eterm*, UWord *I);
Eterm maps_from_list_1(Process*, Eterm*);
Eterm wrap_maps_from_list_1(Process*, Eterm*, UWord *I);
Eterm maps_is_key_2(Process*, Eterm*);
Eterm wrap_maps_is_key_2(Process*, Eterm*, UWord *I);
Eterm maps_keys_1(Process*, Eterm*);
Eterm wrap_maps_keys_1(Process*, Eterm*, UWord *I);
Eterm maps_merge_2(Process*, Eterm*);
Eterm wrap_maps_merge_2(Process*, Eterm*, UWord *I);
Eterm maps_new_0(Process*, Eterm*);
Eterm wrap_maps_new_0(Process*, Eterm*, UWord *I);
Eterm maps_put_3(Process*, Eterm*);
Eterm wrap_maps_put_3(Process*, Eterm*, UWord *I);
Eterm maps_remove_2(Process*, Eterm*);
Eterm wrap_maps_remove_2(Process*, Eterm*, UWord *I);
Eterm maps_update_3(Process*, Eterm*);
Eterm wrap_maps_update_3(Process*, Eterm*, UWord *I);
Eterm maps_values_1(Process*, Eterm*);
Eterm wrap_maps_values_1(Process*, Eterm*, UWord *I);
Eterm erts_internal_cmp_term_2(Process*, Eterm*);
Eterm wrap_erts_internal_cmp_term_2(Process*, Eterm*, UWord *I);
Eterm ets_take_2(Process*, Eterm*);
Eterm wrap_ets_take_2(Process*, Eterm*, UWord *I);
Eterm fun_info_mfa_1(Process*, Eterm*);
Eterm wrap_fun_info_mfa_1(Process*, Eterm*, UWord *I);
Eterm get_keys_0(Process*, Eterm*);
Eterm wrap_get_keys_0(Process*, Eterm*, UWord *I);
Eterm ets_update_counter_4(Process*, Eterm*);
Eterm wrap_ets_update_counter_4(Process*, Eterm*, UWord *I);
Eterm erts_debug_map_info_1(Process*, Eterm*);
Eterm wrap_erts_debug_map_info_1(Process*, Eterm*, UWord *I);
Eterm hash_2(Process*, Eterm*);
Eterm wrap_hash_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_write_u8_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_write_u8_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_write_u32_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_write_u32_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bytearray_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_bytearray_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bytearray_sub_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_bytearray_sub_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bytearray_update_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_bytearray_update_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bitarray_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_bitarray_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bitarray_sub_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_bitarray_sub_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bitarray_update_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_bitarray_update_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_array_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_array_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_array_length_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_array_length_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_array_sub_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_array_sub_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_array_update_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_array_update_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_ref_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_ref_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_ref_get_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_ref_get_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_ref_set_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_ref_set_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_enter_code_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_enter_code_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_alloc_data_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_alloc_data_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_constants_size_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_constants_size_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_merge_term_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_merge_term_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_fun_to_address_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_fun_to_address_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_set_native_address_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_set_native_address_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_set_funinfo_native_address_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_set_funinfo_native_address_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_invalidate_funinfo_native_addresses_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_invalidate_funinfo_native_addresses_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_update_code_size_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_update_code_size_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_code_size_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_code_size_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_enter_sdesc_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_enter_sdesc_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_bif_address_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_bif_address_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_primop_address_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_primop_address_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_atom_to_word_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_atom_to_word_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_term_to_word_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_term_to_word_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_get_fe_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_get_fe_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_set_native_address_in_fe_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_set_native_address_in_fe_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_find_na_or_make_stub_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_find_na_or_make_stub_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_check_crc_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_check_crc_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_system_crc_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_system_crc_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_get_rts_param_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_get_rts_param_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_patch_insn_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_patch_insn_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_patch_call_3(Process*, Eterm*);
Eterm wrap_hipe_bifs_patch_call_3(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_add_ref_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_add_ref_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_mark_referred_from_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_mark_referred_from_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_remove_refs_from_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_remove_refs_from_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_redirect_referred_from_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_redirect_referred_from_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_call_count_on_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_call_count_on_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_call_count_off_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_call_count_off_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_call_count_get_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_call_count_get_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_call_count_clear_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_call_count_clear_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_trap_count_get_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_trap_count_get_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_trap_count_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_trap_count_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_process_info_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_process_info_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_process_info_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_process_info_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_message_info_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_message_info_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_message_info_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_message_info_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_message_sizes_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_message_sizes_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_gc_info_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_gc_info_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_shared_gc_info_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_shared_gc_info_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_incremental_gc_info_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_incremental_gc_info_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_gc_info_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_gc_info_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_pause_times_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_pause_times_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_system_timer_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_system_timer_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_system_timer_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_system_timer_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_send_timer_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_send_timer_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_send_timer_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_send_timer_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_gc_timer_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_gc_timer_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_shared_gc_timer_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_shared_gc_timer_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_gc_timer_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_gc_timer_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_misc_timer_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_misc_timer_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_misc_timer_clear_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_misc_timer_clear_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_get_hrvtime_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_get_hrvtime_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_stop_hrvtime_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_stop_hrvtime_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_show_estack_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_show_estack_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_show_heap_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_show_heap_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_show_nstack_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_show_nstack_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_nstack_used_size_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_nstack_used_size_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_show_pcb_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_show_pcb_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_show_term_1(Process*, Eterm*);
Eterm wrap_hipe_bifs_show_term_1(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_in_native_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_in_native_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_modeswitch_debug_on_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_modeswitch_debug_on_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_modeswitch_debug_off_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_modeswitch_debug_off_0(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_debug_native_called_2(Process*, Eterm*);
Eterm wrap_hipe_bifs_debug_native_called_2(Process*, Eterm*, UWord *I);
Eterm hipe_bifs_llvm_fix_pinned_regs_0(Process*, Eterm*);
Eterm wrap_hipe_bifs_llvm_fix_pinned_regs_0(Process*, Eterm*, UWord *I);
#endif
# 1549 "armv6-portbld-freebsd11.0/erl_bif_table.h"
# 522 "beam/bif.h" 2

#endif
# 524 "beam/bif.h"
# 32 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "big.h"
#endif /* expanded by -frewrite-includes */
# 32 "beam/beam_emu.c"
# 1 "beam/big.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __BIG_H__
#define __BIG_H__

#ifndef __SYS_H__
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/big.h"
# 26 "beam/big.h"
#endif
# 27 "beam/big.h"

#ifndef __CONFIG_H__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 29 "beam/big.h"
# 30 "beam/big.h"
#endif
# 31 "beam/big.h"

#ifndef __GLOBAL_H__
#if 0 /* expanded by -frewrite-includes */
#include "global.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/big.h"
# 34 "beam/big.h"
#endif
# 35 "beam/big.h"

typedef Uint     ErtsDigit;

#if ((SIZEOF_VOID_P == 4) || HALFWORD_HEAP) && defined(SIZEOF_LONG_LONG) && (SIZEOF_LONG_LONG == 8)
/* Assume 32-bit machine with long long support */
typedef Uint64   ErtsDoubleDigit;
typedef Uint16   ErtsHalfDigit;
#define BIG_HAVE_DOUBLE_DIGIT 1

#elif (SIZEOF_VOID_P == 4)
# 45 "beam/big.h"
/* Assume 32-bit machine with no long support */
#undef  BIG_HAVE_DOUBLE_DIGIT
typedef Uint16   ErtsHalfDigit;

#elif (SIZEOF_VOID_P == 8)
# 50 "beam/big.h"
/* Assume 64-bit machine, does it exist 128 bit long long long ? */
#undef  BIG_HAVE_DOUBLE_DIGIT
typedef Uint32   ErtsHalfDigit;
#else
# 54 "beam/big.h"
#error "can not determine machine size"
#endif
# 56 "beam/big.h"

#define D_DECIMAL_EXP	9
#define D_DECIMAL_BASE	1000000000

typedef Uint  dsize_t;	 /* Vector size type */

#define D_EXP (ERTS_SIZEOF_ETERM*8)
#define D_MASK     ((ErtsDigit)(-1))      /* D_BASE-1 */

/* macros for bignum objects */
#define big_v(x)       BIG_V(big_val(x))
#define big_sign(x)    BIG_SIGN(big_val(x))
#define big_arity(x)   BIG_ARITY(big_val(x))
#define big_digit(x,i) BIG_DIGIT(big_val(x),i)
#define big_size(x)    BIG_SIZE(big_val(x))


/* macros for thing pointers */

#define BIG_V(xp)        ((ErtsDigit*)((xp)+1))
#define BIG_SIGN(xp)     (!!bignum_header_is_neg(*xp))
#define BIG_ARITY(xp)    ((Uint)bignum_header_arity(*(xp)))
#define BIG_DIGIT(xp,i)  *(BIG_V(xp)+(i))
#define BIG_DIGITS_PER_WORD (sizeof(Uint)/sizeof(ErtsDigit))

#define BIG_SIZE(xp)  BIG_ARITY(xp)

/* Check for small */
#define IS_USMALL(sgn,x)  ((sgn) ? ((x) <= MAX_SMALL+1) : ((x) <= MAX_SMALL))
#define IS_SSMALL(x)      (((x) >= MIN_SMALL) && ((x) <= MAX_SMALL))

/* The heap size needed for a bignum */
#define BIG_NEED_SIZE(x)  ((x) + 1)
#define BIG_NEED_FOR_BITS(bits) BIG_NEED_SIZE(((bits)-1)/D_EXP + 1)

#define BIG_UINT_HEAP_SIZE (1 + 1)	/* always, since sizeof(Uint) <= sizeof(Eterm) */

#if HALFWORD_HEAP
#define BIG_UWORD_HEAP_SIZE(UW) (((UW) >> (sizeof(Uint) * 8)) ? 3 : 2)
#else
# 96 "beam/big.h"
#define BIG_UWORD_HEAP_SIZE(UW) BIG_UINT_HEAP_SIZE
#endif
# 98 "beam/big.h"

#if defined(ARCH_32) || HALFWORD_HEAP

#define ERTS_UINT64_BIG_HEAP_SIZE__(X) \
  ((X) >= (((Uint64) 1) << 32) ? (1 + 2) : (1 + 1))
#define ERTS_SINT64_HEAP_SIZE(X)				\
  (IS_SSMALL((X))						\
   ? 0								\
   : ERTS_UINT64_BIG_HEAP_SIZE__((X) >= 0 ? (X) : -(Uint64)(X)))
#define ERTS_UINT64_HEAP_SIZE(X)				\
  (IS_USMALL(0, (X)) ? 0 : ERTS_UINT64_BIG_HEAP_SIZE__((X)))
#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 2)
#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 2)
#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) (2*(LEN)+1)

#else
# 114 "beam/big.h"

#define ERTS_SINT64_HEAP_SIZE(X)				\
  (IS_SSMALL((X)) ? 0 : (1 + 1))
#define ERTS_UINT64_HEAP_SIZE(X)				\
  (IS_USMALL(0, (X)) ? 0 : (1 + 1))
#define ERTS_MAX_SINT64_HEAP_SIZE (1 + 1)
#define ERTS_MAX_UINT64_HEAP_SIZE (1 + 1)
#define ERTS_UINT64_ARRAY_TO_BIG_MAX_HEAP_SZ(LEN) ((LEN)+1)

#endif
# 124 "beam/big.h"

int big_decimal_estimate(Wterm);
Eterm erts_big_to_list(Eterm, Eterm**);
char *erts_big_to_string(Wterm x, char *buf, Uint buf_sz);
Uint erts_big_to_binary_bytes(Eterm x, char *buf, Uint buf_sz);

Eterm small_times(Sint, Sint, Eterm*);

Eterm big_plus(Wterm, Wterm, Eterm*);
Eterm big_minus(Eterm, Eterm, Eterm*);
Eterm big_times(Eterm, Eterm, Eterm*);
Eterm big_div(Eterm, Eterm, Eterm*);
Eterm big_rem(Eterm, Eterm, Eterm*);
Eterm big_neg(Eterm, Eterm*);

Eterm big_minus_small(Eterm, Uint, Eterm*);
Eterm big_plus_small(Eterm, Uint, Eterm*);
Eterm big_times_small(Eterm, Uint, Eterm*);

Eterm big_band(Eterm, Eterm, Eterm*);
Eterm big_bor(Eterm, Eterm, Eterm*);
Eterm big_bxor(Eterm, Eterm, Eterm*);
Eterm big_bnot(Eterm, Eterm*);

Eterm big_lshift(Eterm, Sint, Eterm*);
int big_comp (Wterm, Wterm);
int big_ucomp (Eterm, Eterm);
int big_to_double(Wterm x, double* resp);
Eterm double_to_big(double, Eterm*, Uint hsz);
Eterm small_to_big(Sint, Eterm*);
Eterm uint_to_big(Uint, Eterm*);
Eterm uword_to_big(UWord, Eterm*);
Eterm erts_make_integer(Uint, Process *);
Eterm erts_make_integer_from_uword(UWord x, Process *p);

dsize_t big_bytes(Eterm);
Eterm bytes_to_big(byte*, dsize_t, int, Eterm*);
byte* big_to_bytes(Eterm, byte*);

int term_to_Uint(Eterm, Uint*);
int term_to_Uint_mask(Eterm, Uint*);
int term_to_UWord(Eterm, UWord*);
int term_to_Sint(Eterm, Sint*);
#if HAVE_INT64
Eterm erts_uint64_array_to_big(Uint **, int, int, Uint64 *);
int term_to_Uint64(Eterm, Uint64*);
int term_to_Sint64(Eterm, Sint64*);
#endif
# 172 "beam/big.h"

Uint32 big_to_uint32(Eterm b);
int term_equals_2pow32(Eterm);

Eterm erts_uint64_to_big(Uint64, Eterm **);
Eterm erts_sint64_to_big(Sint64, Eterm **);

Eterm erts_chars_to_integer(Process *, char*, Uint, const int);

#endif
# 182 "beam/big.h"

# 33 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "beam_load.h"
#endif /* expanded by -frewrite-includes */
# 33 "beam/beam_emu.c"
# 34 "beam/beam_emu.c"
#if 0 /* expanded by -frewrite-includes */
#include "erl_binary.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/beam_emu.c"
# 1 "beam/erl_binary.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2000-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __ERL_BINARY_H
#define __ERL_BINARY_H

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/erl_binary.h"
# 1 "beam/erl_threads.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/* Description: Error checking thread interface to the ethread library.
 *              All functions terminates the emulator on failure.
 * Author: Rickard Green
 */

#ifndef ERL_THREAD_H__
#define ERL_THREAD_H__

/*
 * --- Documentation of atomics and memory barriers --------------------------
 *
 * The following explicit memory barriers exist:
 *   
 * - ERTS_THR_MEMORY_BARRIER
 *      Full memory barrier. Orders both loads, and stores. No
 *      load or store is allowed to be reordered over the
 *      barrier.
 * - ERTS_THR_WRITE_MEMORY_BARRIER
 *      Write barrier. Orders *only* stores. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_READ_MEMORY_BARRIER
 *      Read barrier. Orders *only* loads. These are not
 *      allowed to be reordered over the barrier.
 * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 *      Data dependency read barrier. Orders *only* loads
 *      according to data dependency across the barrier.
 *
 * If thread support has been disabled, these barriers will become no-ops.
 *
 * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
 * be enabled only in the SMP enabled runtime system.
 *
 * --- Atomic operations ---
 *
 * Atomics operations exist for 32-bit, word size, and double word size
 * integers. Function prototypes are listed below.
 *
 * Each function implementing an atomic operation exist with the following
 * implied memory barrier semantics. Not all combinations are useful, but
 * all of them exist for simplicity. <B> is suffix in function name:
 *
 * - <B>  - Description
 *
 * - mb   - Full memory barrier. Orders both loads, and
 *          stores before, and after the atomic operation.
 *          No load or store is allowed to be reordered
 *          over the atomic operation.
 * - relb - Release barrier. Orders both loads, and
 *          stores appearing *before* the atomic
 *          operation. These are not allowed to be
 *          reordered over the atomic operation.
 * - acqb - Acquire barrier. Orders both loads, and stores
 *          appearing *after* the atomic operation. These
 *          are not allowed to be reordered over the
 *          atomic operation.
 * - wb   - Write barrier. Orders *only* stores. These are
 *          not allowed to be reordered over the barrier.
 *          Store in atomic operation is ordered *after*
 *          the barrier.
 * - rb   - Read barrier. Orders *only* loads. These are
 *          not allowed to be reordered over the barrier.
 *          Load in atomic operation is ordered *before*
 *          the barrier. 
 * - ddrb - Data dependency read barrier. Orders *only*
 *          loads according to data dependency across the
 *          barrier. Load in atomic operation is ordered
 *          before the barrier.
 *
 * If thread support has been disabled, these functions are mapped to
 * functions that performs the same operation, but aren't atomic
 * and don't imply any memory barriers.
 *
 * If the atomic operations are prefixed with erts_smp_ instead of only
 * erts_ the atomic operations will only be atomic in the SMP enabled
 * runtime system, and will be mapped to non-atomic operations without
 * memory barriers in the runtime system without SMP support. Atomic
 * operations with erts_smp_ prefix should use the atomic types
 * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
 * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
 * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
 * are the same.
 *
 * --- 32-bit atomic operations ---
 *
 * The following 32-bit atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above). Note
 * that sizeof(erts_atomic32_t) might be larger than 4!
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
 *                               erts_aint32_t val);
 *
 * Set value:
 *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Read; returns current value:
 *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
 *
 * Increment:
 *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
 *
 * Decrement:
 *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Add value:
 *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
 *                              erts_aint32_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
 *                                            erts_aint32_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
 *                                             erts_aint32_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
 *                                        erts_aint32_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
 *                                           erts_aint32_t new,
 *                                           erts_aint32_t exp);
 *
 * --- Word size atomic operations ---
 *
 * The following word size (same size as sizeof(void *)) atomic
 * operations exist. <B> should be replaced with a supported
 * memory barrier (see above). Note that sizeof(erts_atomic_t)
 * might be larger than sizeof(void *)!
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
 *                             erts_aint_t val);
 *
 * Set value;
 *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Read; returns current value:
 *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
 *
 * Increment; returns resulting value:
 *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
 *
 * Decrement; returns resulting value:
 *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
 *
 * Increment:
 *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
 *
 * Decrement:
 *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
 *
 * Add value; returns resulting value:
 *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Add value:
 *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
 *                            erts_aint_t val);
 *
 * Bitwise-or; returns previous value:
 *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
 *                                        erts_aint_t val);
 *
 * Bitwise-and; returns previous value:
 *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
 *                                         erts_aint_t val);
 *
 * Exchange; returns previous value:
 *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
 *                                    erts_aint_t val);
 *
 * Compare and exchange; returns previous or current value. If
 * returned value equals 'exp' the value was changed to 'new';
 * otherwise not:
 *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
 *                                       erts_aint_t new,
 *                                       erts_aint_t exp);
 *
 * --- Double word size atomic operations ---
 *
 * The following double word atomic operations exist. <B> should be
 * replaced with a supported memory barrier (see above).
 *
 * Note that sizeof(erts_dw_atomic_t) usually is larger than
 * 2*sizeof(void *)!
 * 
 * The erts_dw_aint_t data type should be accessed as if it was defined
 * like this:
 *
 *     typedef struct {
 *         erts_aint_t sint[2];
 *     } erts_dw_aint_t;
 *
 *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
 *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
 *
 *
 * Initialize (not necessarily the same as the set operation):
 *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
 *                           erts_dw_aint_t *val);
 *
 * Set; value is written into 'val':
 *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
 *                               erts_dw_aint_t *val);
 *
 * Read; value is written into 'val':
 *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
 *                                erts_dw_aint_t *val);
 *
 * Compare and exchange; returns a value != 0 if exchange was
 * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
 * contains the same value as in memory when the function is called,
 * 'new' is written to memory; otherwise, not. If exchange was not
 * made, 'exp_act' contains the actual value in memory:
 *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
 *                                  erts_dw_aint_t *new,
 *                                  erts_dw_aint_t *exp_act);
 */

#define ERTS_SPIN_BODY ETHR_SPIN_BODY

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 260 "beam/erl_threads.h"
# 261 "beam/erl_threads.h"

#ifdef USE_THREADS

#define ETHR_TRY_INLINE_FUNCS
#if 0 /* expanded by -frewrite-includes */
#include "ethread.h"
#endif /* expanded by -frewrite-includes */
# 265 "beam/erl_threads.h"
# 266 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_check.h"
#endif /* expanded by -frewrite-includes */
# 266 "beam/erl_threads.h"
# 267 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_lock_count.h"
#endif /* expanded by -frewrite-includes */
# 267 "beam/erl_threads.h"
# 268 "beam/erl_threads.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_term.h"
#endif /* expanded by -frewrite-includes */
# 268 "beam/erl_threads.h"
# 269 "beam/erl_threads.h"

#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4
/*
 * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have
 * only seen this bug in glibc versions before 2.4. Note that condition
 * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by
 * this bug since these implementations may use mutexes internally.
 */
#  define ERTS_THR_HAVE_BUSY_DESTROY_BUG
#endif
# 279 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER

#ifdef ERTS_ENABLE_LOCK_POSITION
#define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
#define erts_mtx_trylock(L) erts_mtx_trylock_x(L, __FILE__, __LINE__)
#define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrlock(L) erts_rwmtx_tryrlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rlock(L) erts_rwmtx_rlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_tryrwlock(L) erts_rwmtx_tryrwlock_x(L, __FILE__, __LINE__)
#define erts_rwmtx_rwlock(L) erts_rwmtx_rwlock_x(L, __FILE__, __LINE__)
#define erts_read_lock(L) erts_read_lock_x(L, __FILE__, __LINE__)
#define erts_write_lock(L) erts_write_lock_x(L, __FILE__, __LINE__)
#endif
# 296 "beam/erl_threads.h"

#define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER
typedef ethr_thr_opts erts_thr_opts_t;
typedef ethr_init_data erts_thr_init_data_t;
typedef ethr_late_init_data erts_thr_late_init_data_t;
typedef ethr_tid erts_tid_t;

/* mutex */
typedef struct {
    ethr_mutex mtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 309 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 312 "beam/erl_threads.h"

} erts_mtx_t;
typedef ethr_cond erts_cnd_t;

/* rwmutex */
typedef struct {
    ethr_rwmutex rwmtx;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 322 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 325 "beam/erl_threads.h"
} erts_rwmtx_t;

#define ERTS_MTX_OPT_DEFAULT_INITER ETHR_MUTEX_OPT_DEFAULT_INITER
#define ERTS_CND_OPT_DEFAULT_INITER ETHR_COND_OPT_DEFAULT_INITER
#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER
#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL
#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \
  ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED
#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED
#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED
typedef ethr_rwmutex_opt erts_rwmtx_opt_t;

typedef ethr_tsd_key erts_tsd_key_t;
typedef ethr_ts_event erts_tse_t;
#define erts_dw_aint_t ethr_dw_sint_t
#define erts_dw_atomic_t ethr_dw_atomic_t
#define erts_aint_t ethr_sint_t
#define erts_atomic_t ethr_atomic_t
#define erts_aint32_t ethr_sint32_t 
#define erts_atomic32_t ethr_atomic32_t

#if defined(ARCH_32)
#  define erts_atomic64_t ethr_dw_atomic_t
#  define erts_aint64_t ethr_sint64_t
#elif defined(ARCH_64)
# 352 "beam/erl_threads.h"
#  define erts_atomic64_t ethr_atomic_t
#  define erts_aint64_t ethr_sint_t
#else
# 355 "beam/erl_threads.h"
#  error "Not supported architecture"
#endif
# 357 "beam/erl_threads.h"

#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD

/* spinlock */
typedef struct {
    ethr_spinlock_t slck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 367 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 370 "beam/erl_threads.h"
} erts_spinlock_t;

/* rwlock */
typedef struct {
    ethr_rwlock_t rwlck;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock_t lc;
#endif
# 378 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_t lcnt;
#endif
# 381 "beam/erl_threads.h"
} erts_rwlock_t;

__decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *); 
                                 /* implemented in erl_init.c */

#define ERTS_THR_INIT_DATA_DEF_INITER	ETHR_INIT_DATA_DEFAULT_INITER
#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \
                                        ETHR_LATE_INIT_DATA_DEFAULT_INITER

#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
#  define ERTS_HAVE_REC_MTX_INIT	ETHR_HAVE_ETHR_REC_MUTEX_INIT
#endif
# 393 "beam/erl_threads.h"

#else /* #ifdef USE_THREADS */
# 395 "beam/erl_threads.h"

#define ERTS_THR_MEMORY_BARRIER
#define ERTS_THR_WRITE_MEMORY_BARRIER
#define ERTS_THR_READ_MEMORY_BARRIER
#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER

#define ERTS_THR_OPTS_DEFAULT_INITER 0
typedef int erts_thr_opts_t;
typedef int erts_thr_init_data_t;
typedef int erts_thr_late_init_data_t;
typedef int erts_tid_t;
typedef int erts_mtx_t;
typedef int erts_cnd_t;
#define ERTS_RWMTX_OPT_DEFAULT_INITER {0}
#define ERTS_RWMTX_TYPE_NORMAL 0
#define ERTS_RWMTX_TYPE_FREQUENT_READ 0
#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0
#define ERTS_RWMTX_LONG_LIVED 0
#define ERTS_RWMTX_SHORT_LIVED 0
#define ERTS_RWMTX_UNKNOWN_LIVED 0
typedef struct {
    char type;
    char lived;
    int main_spincount;
    int aux_spincount;
} erts_rwmtx_opt_t;
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;

typedef struct { SWord sint[2]; } erts_dw_aint_t;
typedef SWord erts_aint_t;
typedef Sint32 erts_aint32_t;
typedef Sint64 erts_aint64_t;

#define erts_dw_atomic_t erts_dw_aint_t
#define erts_atomic_t erts_aint_t
#define erts_atomic32_t erts_aint32_t
#define erts_atomic64_t erts_aint64_t

#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
#else
# 439 "beam/erl_threads.h"
typedef struct { int gcc_is_buggy; } erts_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_rwlock_t;
#endif
# 442 "beam/erl_threads.h"

#ifdef WORDS_BIGENDIAN
#define ERTS_DW_AINT_LOW_WORD 1
#define ERTS_DW_AINT_HIGH_WORD 0
#else
# 447 "beam/erl_threads.h"
#define ERTS_DW_AINT_LOW_WORD 0
#define ERTS_DW_AINT_HIGH_WORD 1
#endif
# 450 "beam/erl_threads.h"

#define ERTS_MTX_INITER			0
#define ERTS_CND_INITER			0
#define ERTS_THR_INIT_DATA_DEF_INITER	0

#define ERTS_HAVE_REC_MTX_INIT		1

#endif /* #ifdef USE_THREADS */
# 458 "beam/erl_threads.h"

#define erts_no_dw_atomic_t erts_dw_aint_t
#define erts_no_atomic_t erts_aint_t
#define erts_no_atomic32_t erts_aint32_t
#define erts_no_atomic64_t erts_aint64_t

#define ERTS_AINT_NULL ((erts_aint_t) NULL)

#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1)))
#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))
#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1)))

ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id);
ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *),
				     void *arg, erts_thr_opts_t *opts);
ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res);
ERTS_GLB_INLINE void erts_thr_detach(erts_tid_t tid);
ERTS_GLB_INLINE void erts_thr_exit(void *res);
ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
				     int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
					 Uint16 opt, int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
					    char *name,
					    Eterm extra,
					    int enable_lcnt);
ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_mtx_trylock_x(erts_mtx_t *mtx, char *file,
				       unsigned int line);
ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file,
				     unsigned int line);
#else
# 499 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_mtx_lock(erts_mtx_t *mtx);
#endif
# 502 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_mtx_is_locked(erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_init(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx);
ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd);
ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no);
ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
					   erts_rwmtx_opt_t *opt,
					   char *name,
					   Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
				       char *name,
				       Eterm extra);
ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
					 erts_rwmtx_opt_t *opt,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx,
					 char *name);
ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE int erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line);
#else
# 529 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
#endif
# 534 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);

ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *val,
					      erts_no_dw_atomic_t *old_val);
ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
						    erts_aint_t i);
ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
						    erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
						     erts_aint_t mask);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
						erts_aint_t new);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
						   erts_aint_t new,
						   erts_aint_t expected);
ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bset(erts_no_atomic_t *var,
						     erts_aint_t mask,
						     erts_aint_t set);
ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
							erts_aint32_t i);
ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
					  erts_aint32_t i);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
							erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
							 erts_aint32_t mask);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
						    erts_aint32_t new);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
						       erts_aint32_t new,
						       erts_aint32_t expected);
ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
							 erts_aint32_t mask,
							 erts_aint32_t set);
ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
							erts_aint64_t i);
ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
					  erts_aint64_t i);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
							erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
							 erts_aint64_t mask);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
						    erts_aint64_t new);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
						       erts_aint64_t new,
						       erts_aint64_t expected);
ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
							 erts_aint64_t mask,
							 erts_aint64_t set);

ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
					      char *name,
					      Eterm extra,
					      Uint16 opt);
ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock,
					  char *name,
					  Eterm extra);
ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock,
					char *name);
ERTS_GLB_INLINE void erts_spinlock_destroy(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line);
#else
# 626 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_spin_lock(erts_spinlock_t *lock);
#endif
# 628 "beam/erl_threads.h"
ERTS_GLB_INLINE int erts_lc_spinlock_is_locked(erts_spinlock_t *lock);
ERTS_GLB_INLINE void erts_rwlock_init_x(erts_rwlock_t *lock,
					char *name,
					Eterm extra);
ERTS_GLB_INLINE void erts_rwlock_init(erts_rwlock_t *lock,
				      char *name);
ERTS_GLB_INLINE void erts_rwlock_destroy(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock);
#ifdef ERTS_ENABLE_LOCK_POSITION
ERTS_GLB_INLINE void erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
ERTS_GLB_INLINE void erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line);
#else
# 640 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_read_lock(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock);
#endif
# 643 "beam/erl_threads.h"
ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock);
ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname);
ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key);
ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value);
ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key);
ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void);
ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep);
ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount);
ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo);
ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep);
ERTS_GLB_INLINE void erts_thr_set_main_status(int, int);
ERTS_GLB_INLINE int erts_thr_get_main_status(void);
ERTS_GLB_INLINE void erts_thr_yield(void);


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS
#define ERTS_THR_HAVE_SIG_FUNCS 1
ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
				      sigset_t *oset);
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);

#ifdef USE_THREADS
ERTS_GLB_INLINE void erts_thr_kill(erts_tid_t tid, int sig);
#endif
# 674 "beam/erl_threads.h"

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 676 "beam/erl_threads.h"

#ifdef USE_THREADS

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
#define ERTS_ATOMIC_BSET_IMPL__(Type, ReadOp, CmpxchgOp, VarP, Mask, Set) \
do {									\
    Type act = ReadOp((VarP));						\
    while (1) {								\
	Type exp = act;							\
	Type new = exp & ~(Mask);					\
	new |= ((Mask) & (Set));					\
	act = CmpxchgOp((VarP), new, exp);				\
	if (act == exp)							\
	    return act;							\
    }									\
} while (0)
#endif
# 750 "beam/erl_threads.h"

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val);
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var);
ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val);
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var);

/*
 * See "Documentation of atomics and memory barriers" at the top
 * of this file for info on atomics.
 */

/* Double word size atomics */

#define erts_dw_atomic_init_nob ethr_dw_atomic_init
#define erts_dw_atomic_set_nob ethr_dw_atomic_set
#define erts_dw_atomic_read_nob ethr_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb

#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb

#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb

#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb

#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb

#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    sint[0] = val->sint[0];
    sint[1] = val->sint[1];    
}

ERTS_GLB_INLINE void
erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    val->sint[0] = sint[0];
    val->sint[1] = sint[1];
}

#endif
# 825 "beam/erl_threads.h"

/* Word size atomics */

#define erts_atomic_init_nob ethr_atomic_init
#define erts_atomic_set_nob ethr_atomic_set
#define erts_atomic_read_nob ethr_atomic_read
#define erts_atomic_inc_read_nob ethr_atomic_inc_read
#define erts_atomic_dec_read_nob ethr_atomic_dec_read
#define erts_atomic_inc_nob ethr_atomic_inc
#define erts_atomic_dec_nob ethr_atomic_dec
#define erts_atomic_add_read_nob ethr_atomic_add_read
#define erts_atomic_add_nob ethr_atomic_add
#define erts_atomic_read_bor_nob ethr_atomic_read_bor
#define erts_atomic_read_band_nob ethr_atomic_read_band
#define erts_atomic_xchg_nob ethr_atomic_xchg
#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_nob(erts_atomic_t *var,
			  erts_aint_t mask,
			  erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg,
			    var, mask, set);
}
#endif
# 854 "beam/erl_threads.h"

#define erts_atomic_init_mb ethr_atomic_init_mb
#define erts_atomic_set_mb ethr_atomic_set_mb
#define erts_atomic_read_mb ethr_atomic_read_mb
#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic_inc_mb ethr_atomic_inc_mb
#define erts_atomic_dec_mb ethr_atomic_dec_mb
#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic_add_mb ethr_atomic_add_mb
#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_mb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 881 "beam/erl_threads.h"

#define erts_atomic_init_acqb ethr_atomic_init_acqb
#define erts_atomic_set_acqb ethr_atomic_set_acqb
#define erts_atomic_read_acqb ethr_atomic_read_acqb
#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic_add_acqb ethr_atomic_add_acqb
#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_acqb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 908 "beam/erl_threads.h"

#define erts_atomic_init_relb ethr_atomic_init_relb
#define erts_atomic_set_relb ethr_atomic_set_relb
#define erts_atomic_read_relb ethr_atomic_read_relb
#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic_inc_relb ethr_atomic_inc_relb
#define erts_atomic_dec_relb ethr_atomic_dec_relb
#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic_add_relb ethr_atomic_add_relb
#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_relb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 935 "beam/erl_threads.h"

#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_ddrb(erts_atomic_t *var,
			   erts_aint_t mask,
			   erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 962 "beam/erl_threads.h"

#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic_inc_rb ethr_atomic_inc_rb
#define erts_atomic_dec_rb ethr_atomic_dec_rb
#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic_add_rb ethr_atomic_add_rb
#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_rb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 989 "beam/erl_threads.h"

#define erts_atomic_init_wb ethr_atomic_init_wb
#define erts_atomic_set_wb ethr_atomic_set_wb
#define erts_atomic_read_wb ethr_atomic_read_wb
#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic_inc_wb ethr_atomic_inc_wb
#define erts_atomic_dec_wb ethr_atomic_dec_wb
#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic_add_wb ethr_atomic_add_wb
#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_bset_wb(erts_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint_t,
			    ethr_atomic_read,
			    ethr_atomic_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1018 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint_t
erts_atomic_read_dirty(erts_atomic_t *var)
{
    ethr_sint_t *sint = ethr_atomic_addr(var);
    return *sint;
}

#endif
# 1036 "beam/erl_threads.h"

/* 32-bit atomics */

#define erts_atomic32_init_nob ethr_atomic32_init
#define erts_atomic32_set_nob ethr_atomic32_set
#define erts_atomic32_read_nob ethr_atomic32_read
#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
#define erts_atomic32_inc_nob ethr_atomic32_inc
#define erts_atomic32_dec_nob ethr_atomic32_dec
#define erts_atomic32_add_read_nob ethr_atomic32_add_read
#define erts_atomic32_add_nob ethr_atomic32_add
#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
#define erts_atomic32_read_band_nob ethr_atomic32_read_band
#define erts_atomic32_xchg_nob ethr_atomic32_xchg
#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_nob(erts_atomic32_t *var,
			    erts_aint32_t mask,
			    erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg,
			    var, mask, set);
}
#endif
# 1065 "beam/erl_threads.h"

#define erts_atomic32_init_mb ethr_atomic32_init_mb
#define erts_atomic32_set_mb ethr_atomic32_set_mb
#define erts_atomic32_read_mb ethr_atomic32_read_mb
#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
#define erts_atomic32_add_mb ethr_atomic32_add_mb
#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_mb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_mb,
			    var, mask, set);
}
#endif
# 1092 "beam/erl_threads.h"

#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_acqb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_acqb,
			    var, mask, set);
}
#endif
# 1119 "beam/erl_threads.h"

#define erts_atomic32_init_relb ethr_atomic32_init_relb
#define erts_atomic32_set_relb ethr_atomic32_set_relb
#define erts_atomic32_read_relb ethr_atomic32_read_relb
#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
#define erts_atomic32_add_relb ethr_atomic32_add_relb
#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_relb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_relb,
			    var, mask, set);
}
#endif
# 1146 "beam/erl_threads.h"

#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_ddrb(erts_atomic32_t *var,
			     erts_aint32_t mask,
			     erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_ddrb,
			    var, mask, set);
}
#endif
# 1173 "beam/erl_threads.h"

#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
#define erts_atomic32_add_rb ethr_atomic32_add_rb
#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_rb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_rb,
			    var, mask, set);
}
#endif
# 1200 "beam/erl_threads.h"

#define erts_atomic32_init_wb ethr_atomic32_init_wb
#define erts_atomic32_set_wb ethr_atomic32_set_wb
#define erts_atomic32_read_wb ethr_atomic32_read_wb
#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
#define erts_atomic32_add_wb ethr_atomic32_add_wb
#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_bset_wb(erts_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    ERTS_ATOMIC_BSET_IMPL__(erts_aint32_t,
			    ethr_atomic32_read,
			    ethr_atomic32_cmpxchg_wb,
			    var, mask, set);
}

#endif
# 1229 "beam/erl_threads.h"

#undef ERTS_ATOMIC_BSET_IMPL__

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    *sint = val;
}

ERTS_GLB_INLINE erts_aint32_t
erts_atomic32_read_dirty(erts_atomic32_t *var)
{
    ethr_sint32_t *sint = ethr_atomic32_addr(var);
    return *sint;
}

#endif
# 1249 "beam/erl_threads.h"

/* 64-bit atomics */

#if defined(ARCH_64)

#define erts_atomic64_init_nob ethr_atomic_init
#define erts_atomic64_set_nob ethr_atomic_set
#define erts_atomic64_read_nob ethr_atomic_read
#define erts_atomic64_inc_read_nob ethr_atomic_inc_read
#define erts_atomic64_dec_read_nob ethr_atomic_dec_read
#define erts_atomic64_inc_nob ethr_atomic_inc
#define erts_atomic64_dec_nob ethr_atomic_dec
#define erts_atomic64_add_read_nob ethr_atomic_add_read
#define erts_atomic64_add_nob ethr_atomic_add
#define erts_atomic64_read_bor_nob ethr_atomic_read_bor
#define erts_atomic64_read_band_nob ethr_atomic_read_band
#define erts_atomic64_xchg_nob ethr_atomic_xchg
#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob

#define erts_atomic64_init_mb ethr_atomic_init_mb
#define erts_atomic64_set_mb ethr_atomic_set_mb
#define erts_atomic64_read_mb ethr_atomic_read_mb
#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
#define erts_atomic64_inc_mb ethr_atomic_inc_mb
#define erts_atomic64_dec_mb ethr_atomic_dec_mb
#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
#define erts_atomic64_add_mb ethr_atomic_add_mb
#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb

#define erts_atomic64_init_acqb ethr_atomic_init_acqb
#define erts_atomic64_set_acqb ethr_atomic_set_acqb
#define erts_atomic64_read_acqb ethr_atomic_read_acqb
#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
#define erts_atomic64_add_acqb ethr_atomic_add_acqb
#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb

#define erts_atomic64_init_relb ethr_atomic_init_relb
#define erts_atomic64_set_relb ethr_atomic_set_relb
#define erts_atomic64_read_relb ethr_atomic_read_relb
#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
#define erts_atomic64_inc_relb ethr_atomic_inc_relb
#define erts_atomic64_dec_relb ethr_atomic_dec_relb
#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
#define erts_atomic64_add_relb ethr_atomic_add_relb
#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb

#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb

#define erts_atomic64_init_rb ethr_atomic_init_rb
#define erts_atomic64_set_rb ethr_atomic_set_rb
#define erts_atomic64_read_rb ethr_atomic_read_rb
#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
#define erts_atomic64_inc_rb ethr_atomic_inc_rb
#define erts_atomic64_dec_rb ethr_atomic_dec_rb
#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
#define erts_atomic64_add_rb ethr_atomic_add_rb
#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb

#define erts_atomic64_init_wb ethr_atomic_init_wb
#define erts_atomic64_set_wb ethr_atomic_set_wb
#define erts_atomic64_read_wb ethr_atomic_read_wb
#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
#define erts_atomic64_inc_wb ethr_atomic_inc_wb
#define erts_atomic64_dec_wb ethr_atomic_dec_wb
#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
#define erts_atomic64_add_wb ethr_atomic_add_wb
#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb

#define erts_atomic64_set_dirty erts_atomic_set_dirty
#define erts_atomic64_read_dirty erts_atomic_read_dirty

#elif defined(ARCH_32)
# 1363 "beam/erl_threads.h"

#undef ERTS_ATOMIC64_OPS_DECL__

#define ERTS_ATOMIC64_OPS_DECL__(BARRIER)				\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var);		\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp);			\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)

ERTS_ATOMIC64_OPS_DECL__(nob);
ERTS_ATOMIC64_OPS_DECL__(mb);
ERTS_ATOMIC64_OPS_DECL__(acqb);
ERTS_ATOMIC64_OPS_DECL__(relb);
ERTS_ATOMIC64_OPS_DECL__(ddrb);
ERTS_ATOMIC64_OPS_DECL__(rb);
ERTS_ATOMIC64_OPS_DECL__(wb);

#undef ERTS_ATOMIC64_OPS_DECL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

/*
 * The ethr_dw_atomic_*_nob() functions below
 * are here to make it possible for the
 * ERTS_ATOMIC64_OPS_IMPL__() to map erts
 * barriers to ethread barriers...
 */
static ERTS_INLINE void
ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_init(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
		       ethr_dw_sint_t *val)
{
    ethr_dw_atomic_set(var, val);
}

static ERTS_INLINE void
ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
			ethr_dw_sint_t *val)
{
    ethr_dw_atomic_read(var, val);
}

static ERTS_INLINE int
ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
			   ethr_dw_sint_t *new,
			   ethr_dw_sint_t *xchg)
{
    return ethr_dw_atomic_cmpxchg(var, new, xchg);
}

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#ifdef ETHR_SU_DW_NAINT_T__
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((erts_aint64_t) DW.dw_sint)
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
#else /* !ETHR_SU_DW_NAINT_T__ */
# 1470 "beam/erl_threads.h"
#define ERTS_DW_SINT_TO_AINT64__(DW)					\
    ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32)		\
     | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD])		\
	& ((erts_aint64_t) 0xffffffff)))
#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
    do {								\
	DW.sint[ETHR_DW_SINT_LOW_WORD] =				\
	    (ethr_sint_t) (AINT64 & 0xffffffff);			\
	DW.sint[ETHR_DW_SINT_HIGH_WORD] = 				\
	    (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff);		\
    } while (0)
#endif /* !ETHR_SU_DW_NAINT_T__ */
# 1482 "beam/erl_threads.h"

#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, 			\
					AVarP, XchgVar, NewVar, 	\
					ModificationCode)		\
do {									\
    ethr_dw_sint_t dw_xchg__, dw_new__;					\
    ethr_dw_atomic_read(AVarP, &dw_xchg__);				\
    do {								\
	XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__);			\
	{								\
	    ModificationCode;						\
	}								\
	ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar);			\
    } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__));		\
} while (0)

#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER)				\
    									\
ERTS_GLB_INLINE void							\
erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_init_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    ethr_dw_sint_t dw;							\
    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
    ethr_dw_atomic_set_ ## BARRIER(var, &dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    ethr_dw_sint_t dw;							\
    ethr_dw_atomic_read_ ## BARRIER(var, &dw);				\
    return ERTS_DW_SINT_TO_AINT64__(dw);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var)		\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + 1);			\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg - 1);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
    return new;								\
}									\
									\
ERTS_GLB_INLINE void							\
erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
			      erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg + val);			\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
				   erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg | val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = xchg & val);			\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
			       erts_aint64_t val)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    new = val);				\
    return xchg;							\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
				  erts_aint64_t new,			\
				  erts_aint64_t exp)			\
{									\
    ethr_dw_sint_t dw_xchg, dw_new;					\
    ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp);				\
    ERTS_AINT64_TO_DW_SINT__(dw_new, new);				\
    if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg))	\
	return exp;							\
    return ERTS_DW_SINT_TO_AINT64__(dw_xchg);				\
}									\
									\
ERTS_GLB_INLINE erts_aint64_t						\
erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
				    erts_aint64_t mask,			\
				    erts_aint64_t set)			\
{									\
    erts_aint64_t xchg, new;						\
    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
				    var, xchg, new,			\
				    {					\
					new = xchg & ~mask;		\
					new |= mask & set;		\
				    });					\
    return xchg;							\
}

ERTS_ATOMIC64_OPS_IMPL__(nob)
ERTS_ATOMIC64_OPS_IMPL__(mb)
ERTS_ATOMIC64_OPS_IMPL__(acqb)
ERTS_ATOMIC64_OPS_IMPL__(relb)
ERTS_ATOMIC64_OPS_IMPL__(ddrb)
ERTS_ATOMIC64_OPS_IMPL__(rb)
ERTS_ATOMIC64_OPS_IMPL__(wb)

#undef ERTS_ATOMIC64_OPS_IMPL__
#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__

ERTS_GLB_INLINE void
erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
{
    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
    ethr_dw_sint_t dw;
    ERTS_AINT64_TO_DW_SINT__(dw, val);
    sint[0] = dw.sint[0];
    sint[1] = dw.sint[1];
}

ERTS_GLB_INLINE erts_aint64_t
erts_atomic64_read_dirty(erts_atomic64_t *var)
{
    ethr_sint_t *sint;
    ethr_dw_sint_t dw;
    sint = ethr_dw_atomic_addr(var);
    dw.sint[0] = sint[0];
    dw.sint[1] = sint[1];
    return ERTS_DW_SINT_TO_AINT64__(dw);
}

#undef ERTS_DW_SINT_TO_AINT64__
#undef ERTS_AINT64_TO_DW_SINT__

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 1682 "beam/erl_threads.h"

#endif /* ARCH_32 */
# 1684 "beam/erl_threads.h"

#else /* !USE_THREADS */
# 1686 "beam/erl_threads.h"

/* Double word size atomics */

#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg

#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set
#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read

/* Word size atomics */

#define erts_atomic_init_nob erts_no_atomic_set
#define erts_atomic_set_nob erts_no_atomic_set
#define erts_atomic_read_nob erts_no_atomic_read
#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
#define erts_atomic_inc_nob erts_no_atomic_inc
#define erts_atomic_dec_nob erts_no_atomic_dec
#define erts_atomic_add_read_nob erts_no_atomic_add_read
#define erts_atomic_add_nob erts_no_atomic_add
#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
#define erts_atomic_read_band_nob erts_no_atomic_read_band
#define erts_atomic_xchg_nob erts_no_atomic_xchg
#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_nob erts_no_atomic_read_bset

#define erts_atomic_init_mb erts_no_atomic_set
#define erts_atomic_set_mb erts_no_atomic_set
#define erts_atomic_read_mb erts_no_atomic_read
#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
#define erts_atomic_inc_mb erts_no_atomic_inc
#define erts_atomic_dec_mb erts_no_atomic_dec
#define erts_atomic_add_read_mb erts_no_atomic_add_read
#define erts_atomic_add_mb erts_no_atomic_add
#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
#define erts_atomic_read_band_mb erts_no_atomic_read_band
#define erts_atomic_xchg_mb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_mb erts_no_atomic_read_bset

#define erts_atomic_init_acqb erts_no_atomic_set
#define erts_atomic_set_acqb erts_no_atomic_set
#define erts_atomic_read_acqb erts_no_atomic_read
#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
#define erts_atomic_inc_acqb erts_no_atomic_inc
#define erts_atomic_dec_acqb erts_no_atomic_dec
#define erts_atomic_add_read_acqb erts_no_atomic_add_read
#define erts_atomic_add_acqb erts_no_atomic_add
#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
#define erts_atomic_read_band_acqb erts_no_atomic_read_band
#define erts_atomic_xchg_acqb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_acqb erts_no_atomic_read_bset

#define erts_atomic_init_relb erts_no_atomic_set
#define erts_atomic_set_relb erts_no_atomic_set
#define erts_atomic_read_relb erts_no_atomic_read
#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
#define erts_atomic_inc_relb erts_no_atomic_inc
#define erts_atomic_dec_relb erts_no_atomic_dec
#define erts_atomic_add_read_relb erts_no_atomic_add_read
#define erts_atomic_add_relb erts_no_atomic_add
#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
#define erts_atomic_read_band_relb erts_no_atomic_read_band
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_relb erts_no_atomic_read_bset

#define erts_atomic_init_ddrb erts_no_atomic_set
#define erts_atomic_set_ddrb erts_no_atomic_set
#define erts_atomic_read_ddrb erts_no_atomic_read
#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
#define erts_atomic_inc_ddrb erts_no_atomic_inc
#define erts_atomic_dec_ddrb erts_no_atomic_dec
#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
#define erts_atomic_add_ddrb erts_no_atomic_add
#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_ddrb erts_no_atomic_read_bset

#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
#define erts_atomic_read_rb erts_no_atomic_read
#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
#define erts_atomic_inc_rb erts_no_atomic_inc
#define erts_atomic_dec_rb erts_no_atomic_dec
#define erts_atomic_add_read_rb erts_no_atomic_add_read
#define erts_atomic_add_rb erts_no_atomic_add
#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
#define erts_atomic_read_band_rb erts_no_atomic_read_band
#define erts_atomic_xchg_rb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_rb erts_no_atomic_read_bset

#define erts_atomic_init_wb erts_no_atomic_set
#define erts_atomic_set_wb erts_no_atomic_set
#define erts_atomic_read_wb erts_no_atomic_read
#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
#define erts_atomic_inc_wb erts_no_atomic_inc
#define erts_atomic_dec_wb erts_no_atomic_dec
#define erts_atomic_add_read_wb erts_no_atomic_add_read
#define erts_atomic_add_wb erts_no_atomic_add
#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
#define erts_atomic_read_band_wb erts_no_atomic_read_band
#define erts_atomic_xchg_wb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
#define erts_atomic_read_bset_wb erts_no_atomic_read_bset

#define erts_atomic_set_dirty erts_no_atomic_set
#define erts_atomic_read_dirty erts_no_atomic_read

/* 32-bit atomics */

#define erts_atomic32_init_nob erts_no_atomic32_set
#define erts_atomic32_set_nob erts_no_atomic32_set
#define erts_atomic32_read_nob erts_no_atomic32_read
#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
#define erts_atomic32_inc_nob erts_no_atomic32_inc
#define erts_atomic32_dec_nob erts_no_atomic32_dec
#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
#define erts_atomic32_add_nob erts_no_atomic32_add
#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_nob erts_no_atomic32_read_bset

#define erts_atomic32_init_mb erts_no_atomic32_set
#define erts_atomic32_set_mb erts_no_atomic32_set
#define erts_atomic32_read_mb erts_no_atomic32_read
#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
#define erts_atomic32_inc_mb erts_no_atomic32_inc
#define erts_atomic32_dec_mb erts_no_atomic32_dec
#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
#define erts_atomic32_add_mb erts_no_atomic32_add
#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_mb erts_no_atomic32_read_bset

#define erts_atomic32_init_acqb erts_no_atomic32_set
#define erts_atomic32_set_acqb erts_no_atomic32_set
#define erts_atomic32_read_acqb erts_no_atomic32_read
#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
#define erts_atomic32_inc_acqb erts_no_atomic32_inc
#define erts_atomic32_dec_acqb erts_no_atomic32_dec
#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
#define erts_atomic32_add_acqb erts_no_atomic32_add
#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_acqb erts_no_atomic32_read_bset

#define erts_atomic32_init_relb erts_no_atomic32_set
#define erts_atomic32_set_relb erts_no_atomic32_set
#define erts_atomic32_read_relb erts_no_atomic32_read
#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
#define erts_atomic32_inc_relb erts_no_atomic32_inc
#define erts_atomic32_dec_relb erts_no_atomic32_dec
#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
#define erts_atomic32_add_relb erts_no_atomic32_add
#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_relb erts_no_atomic32_read_bset

#define erts_atomic32_init_ddrb erts_no_atomic32_set
#define erts_atomic32_set_ddrb erts_no_atomic32_set
#define erts_atomic32_read_ddrb erts_no_atomic32_read
#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
#define erts_atomic32_add_ddrb erts_no_atomic32_add
#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_ddrb erts_no_atomic32_read_bset

#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
#define erts_atomic32_read_rb erts_no_atomic32_read
#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
#define erts_atomic32_inc_rb erts_no_atomic32_inc
#define erts_atomic32_dec_rb erts_no_atomic32_dec
#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
#define erts_atomic32_add_rb erts_no_atomic32_add
#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_rb erts_no_atomic32_read_bset

#define erts_atomic32_init_wb erts_no_atomic32_set
#define erts_atomic32_set_wb erts_no_atomic32_set
#define erts_atomic32_read_wb erts_no_atomic32_read
#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
#define erts_atomic32_inc_wb erts_no_atomic32_inc
#define erts_atomic32_dec_wb erts_no_atomic32_dec
#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
#define erts_atomic32_add_wb erts_no_atomic32_add
#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
#define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset

#define erts_atomic32_set_dirty erts_no_atomic32_set
#define erts_atomic32_read_dirty erts_no_atomic32_read

/* 64-bit atomics */

#define erts_atomic64_init_nob erts_no_atomic64_set
#define erts_atomic64_set_nob erts_no_atomic64_set
#define erts_atomic64_read_nob erts_no_atomic64_read
#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
#define erts_atomic64_inc_nob erts_no_atomic64_inc
#define erts_atomic64_dec_nob erts_no_atomic64_dec
#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
#define erts_atomic64_add_nob erts_no_atomic64_add
#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset

#define erts_atomic64_init_mb erts_no_atomic64_set
#define erts_atomic64_set_mb erts_no_atomic64_set
#define erts_atomic64_read_mb erts_no_atomic64_read
#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
#define erts_atomic64_inc_mb erts_no_atomic64_inc
#define erts_atomic64_dec_mb erts_no_atomic64_dec
#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
#define erts_atomic64_add_mb erts_no_atomic64_add
#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset

#define erts_atomic64_init_acqb erts_no_atomic64_set
#define erts_atomic64_set_acqb erts_no_atomic64_set
#define erts_atomic64_read_acqb erts_no_atomic64_read
#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
#define erts_atomic64_inc_acqb erts_no_atomic64_inc
#define erts_atomic64_dec_acqb erts_no_atomic64_dec
#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
#define erts_atomic64_add_acqb erts_no_atomic64_add
#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset

#define erts_atomic64_init_relb erts_no_atomic64_set
#define erts_atomic64_set_relb erts_no_atomic64_set
#define erts_atomic64_read_relb erts_no_atomic64_read
#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
#define erts_atomic64_inc_relb erts_no_atomic64_inc
#define erts_atomic64_dec_relb erts_no_atomic64_dec
#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
#define erts_atomic64_add_relb erts_no_atomic64_add
#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset

#define erts_atomic64_init_ddrb erts_no_atomic64_set
#define erts_atomic64_set_ddrb erts_no_atomic64_set
#define erts_atomic64_read_ddrb erts_no_atomic64_read
#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
#define erts_atomic64_add_ddrb erts_no_atomic64_add
#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset

#define erts_atomic64_init_rb erts_no_atomic64_set
#define erts_atomic64_set_rb erts_no_atomic64_set
#define erts_atomic64_read_rb erts_no_atomic64_read
#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
#define erts_atomic64_inc_rb erts_no_atomic64_inc
#define erts_atomic64_dec_rb erts_no_atomic64_dec
#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
#define erts_atomic64_add_rb erts_no_atomic64_add
#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset

#define erts_atomic64_init_wb erts_no_atomic64_set
#define erts_atomic64_set_wb erts_no_atomic64_set
#define erts_atomic64_read_wb erts_no_atomic64_read
#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
#define erts_atomic64_inc_wb erts_no_atomic64_inc
#define erts_atomic64_dec_wb erts_no_atomic64_dec
#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
#define erts_atomic64_add_wb erts_no_atomic64_add
#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset

#define erts_atomic64_set_dirty erts_no_atomic64_set
#define erts_atomic64_read_dirty erts_no_atomic64_read

#endif /* !USE_THREADS */
# 2058 "beam/erl_threads.h"

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_thr_init(erts_thr_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_init(id);
    if (res)
	erts_thr_fatal_error(res, "initialize thread library");
#endif
# 2069 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_late_init(erts_thr_late_init_data_t *id)
{
#ifdef USE_THREADS
    int res = ethr_late_init(id);
    if (res)
	erts_thr_fatal_error(res, "complete initialization of thread library");
#endif
# 2079 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg,
		erts_thr_opts_t *opts)
{
#ifdef USE_THREADS
    int res = ethr_thr_create(tid, func, arg, opts);
    if (res)
	erts_thr_fatal_error(res, "create thread");
#endif
# 2090 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_join(erts_tid_t tid, void **thr_res)
{
#ifdef USE_THREADS
    int res = ethr_thr_join(tid, thr_res);
    if (res)
	erts_thr_fatal_error(res, "join thread");
#endif
# 2100 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_detach(erts_tid_t tid)
{
#ifdef USE_THREADS
    int res = ethr_thr_detach(tid);
    if (res)
	erts_thr_fatal_error(res, "detach thread");
#endif
# 2111 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_thr_exit(void *res)
{
#ifdef USE_THREADS
    ethr_thr_exit(res);
    erts_thr_fatal_error(0, "terminate thread");
#endif
# 2121 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_install_exit_handler(void (*exit_handler)(void))
{
#ifdef USE_THREADS
    int res = ethr_install_exit_handler(exit_handler);
    if (res != 0)
	erts_thr_fatal_error(res, "install thread exit handler");
#endif
# 2131 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tid_t
erts_thr_self(void)
{
#ifdef USE_THREADS
    return ethr_self();
#else
# 2139 "beam/erl_threads.h"
    return 0;
#endif
# 2141 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_thr_getname(erts_tid_t tid, char *buf, size_t len)
{
#ifdef USE_THREADS
    return ethr_getname(tid, buf, len);
#else
# 2149 "beam/erl_threads.h"
    return -1;
#endif
# 2151 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
erts_equal_tids(erts_tid_t x, erts_tid_t y)
{
#ifdef USE_THREADS
    return ethr_equal_tids(x, y);
#else
# 2160 "beam/erl_threads.h"
    return 1;
#endif
# 2162 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2174 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2180 "beam/erl_threads.h"
#endif
# 2181 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
		    int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2194 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
#endif
# 2200 "beam/erl_threads.h"
#endif
# 2201 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
#endif
# 2214 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (enable_lcnt)
      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
    else
      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
#endif
# 2220 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2224 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2227 "beam/erl_threads.h"
#endif
# 2228 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2240 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2243 "beam/erl_threads.h"
#endif
# 2244 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_init_locked(erts_mtx_t *mtx, char *name)
{
#ifdef USE_THREADS
    int res = ethr_mutex_init(&mtx->mtx);
    if (res)
	erts_thr_fatal_error(res, "initialize mutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX);
#endif
# 2256 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX);
#endif
# 2259 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_trylock(1, &mtx->lc);
#endif
# 2263 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, 1);
#endif
# 2266 "beam/erl_threads.h"
#endif
# 2267 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_destroy(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&mtx->lc);
#endif
# 2277 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&mtx->lcnt);
#endif
# 2280 "beam/erl_threads.h"
    res = ethr_mutex_destroy(&mtx->mtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy mutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2290 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy mutex");
    }
#endif
# 2293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_trylock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2299 "beam/erl_threads.h"
erts_mtx_trylock(erts_mtx_t *mtx)
#endif
# 2301 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy(&mtx->lc))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2310 "beam/erl_threads.h"

    res = ethr_mutex_trylock(&mtx->mtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_x(res == 0, &mtx->lc,file,line);
#else
# 2317 "beam/erl_threads.h"
    erts_lc_trylock(res == 0, &mtx->lc);
#endif
# 2319 "beam/erl_threads.h"
#endif
# 2320 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock(&mtx->lcnt, res);
#endif    
# 2323 "beam/erl_threads.h"
    return res;
#else
# 2325 "beam/erl_threads.h"
    return 0;
#endif
# 2327 "beam/erl_threads.h"

}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line)
#else
# 2334 "beam/erl_threads.h"
erts_mtx_lock(erts_mtx_t *mtx)
#endif
# 2336 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&mtx->lc, file, line);
#else
# 2342 "beam/erl_threads.h"
    erts_lc_lock(&mtx->lc);
#endif
# 2344 "beam/erl_threads.h"
#endif
# 2345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2348 "beam/erl_threads.h"
    ethr_mutex_lock(&mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&mtx->lcnt, file, line);
#endif
# 2352 "beam/erl_threads.h"
#endif
# 2353 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_mtx_unlock(erts_mtx_t *mtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2362 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2365 "beam/erl_threads.h"
    ethr_mutex_unlock(&mtx->mtx);
#endif
# 2367 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_mtx_is_locked(erts_mtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2379 "beam/erl_threads.h"
    return 0;
#endif
# 2381 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_init(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_init(cnd);
    if (res)
	erts_thr_fatal_error(res, "initialize condition variable");
#endif
# 2391 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_destroy(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    int res = ethr_cond_destroy(cnd);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy cond destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2407 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy condition variable");
    }
#endif
# 2410 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&mtx->lc);
#endif
# 2420 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&mtx->lcnt);
#endif
# 2423 "beam/erl_threads.h"
    res = ethr_cond_wait(cnd, &mtx->mtx);
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_lock(&mtx->lc);
#endif
# 2427 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&mtx->lcnt);
#endif
# 2430 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post(&mtx->lcnt);
#endif
# 2433 "beam/erl_threads.h"
    if (res != 0 && res != EINTR)
	erts_thr_fatal_error(res, "wait on condition variable");
#endif
# 2436 "beam/erl_threads.h"
}

/*
 * IMPORTANT note about erts_cnd_signal() and erts_cnd_broadcast()
 *
 * POSIX allow a call to `pthread_cond_signal' or `pthread_cond_broadcast'
 * even though the associated mutex/mutexes isn't/aren't locked by the
 * caller. Our implementation do not allow that in order to avoid a
 * performance penalty. That is, all associated mutexes *need* to be
 * locked by the caller of erts_cnd_signal()/erts_cnd_broadcast()!
 */

ERTS_GLB_INLINE void
erts_cnd_signal(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_signal(cnd);
#endif
# 2454 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_cnd_broadcast(erts_cnd_t *cnd)
{
#ifdef USE_THREADS
    ethr_cond_broadcast(cnd);
#endif
# 2463 "beam/erl_threads.h"
}

/* rwmutex */

ERTS_GLB_INLINE void
erts_rwmtx_set_reader_group(int no)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2475 "beam/erl_threads.h"
    res = ethr_rwmutex_set_reader_group(no);
    if (res != 0)
	erts_thr_fatal_error(res, "set reader group");
#endif
# 2479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx,
		      erts_rwmtx_opt_t *opt,
		      char *name,
		      Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra);
#endif
# 2494 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    if (name && name[0] == '\0')
      erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra);
    else
      erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra);
#endif
# 2500 "beam/erl_threads.h"
#endif
# 2501 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init_x(erts_rwmtx_t *rwmtx,
		  char *name,
		  Eterm extra)
{
    erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra);
}

ERTS_GLB_INLINE void
erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx,
		    erts_rwmtx_opt_t *opt,
		    char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt);
    if (res != 0)
	erts_thr_fatal_error(res, "initialize rwmutex");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX);
#endif
# 2523 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX);
#endif
# 2526 "beam/erl_threads.h"
#endif
# 2527 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name)
{
    erts_rwmtx_init_opt(rwmtx, NULL, name);
}

ERTS_GLB_INLINE void
erts_rwmtx_destroy(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&rwmtx->lc);
#endif
# 2543 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&rwmtx->lcnt);
#endif
# 2546 "beam/erl_threads.h"
    res = ethr_rwmutex_destroy(&rwmtx->rwmtx);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwmutex destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 2556 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwmutex");
    }
#endif
# 2559 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2565 "beam/erl_threads.h"
erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx)
#endif
# 2567 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2576 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2583 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2585 "beam/erl_threads.h"
#endif
# 2586 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ);
#endif
# 2589 "beam/erl_threads.h"
    
    return res;
#else
# 2592 "beam/erl_threads.h"
    return 0;
#endif
# 2594 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2600 "beam/erl_threads.h"
erts_rwmtx_rlock(erts_rwmtx_t *rwmtx)
#endif
# 2602 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 2608 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2610 "beam/erl_threads.h"
#endif
# 2611 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2614 "beam/erl_threads.h"
    ethr_rwmutex_rlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2618 "beam/erl_threads.h"
#endif
# 2619 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_runlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ);
#endif
# 2628 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ);
#endif
# 2631 "beam/erl_threads.h"
    ethr_rwmutex_runlock(&rwmtx->rwmtx);
#endif
# 2633 "beam/erl_threads.h"
}


ERTS_GLB_INLINE int
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_tryrwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2640 "beam/erl_threads.h"
erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx)
#endif
# 2642 "beam/erl_threads.h"
{
#ifdef USE_THREADS
    int res;

#ifdef ERTS_ENABLE_LOCK_CHECK
    if (erts_lc_trylock_force_busy_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE))
	return EBUSY; /* Make sure caller can handle the situation without
			 causing a lock order violation */
#endif
# 2651 "beam/erl_threads.h"

    res = ethr_rwmutex_tryrwlock(&rwmtx->rwmtx);

#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_trylock_flg_x(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2658 "beam/erl_threads.h"
    erts_lc_trylock_flg(res == 0, &rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2660 "beam/erl_threads.h"
#endif
# 2661 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2664 "beam/erl_threads.h"
    
    return res;
#else
# 2667 "beam/erl_threads.h"
    return 0;
#endif
# 2669 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_rwmtx_rwlock_x(erts_rwmtx_t *rwmtx, char *file, unsigned int line)
#else
# 2675 "beam/erl_threads.h"
erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx)
#endif
# 2677 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 2683 "beam/erl_threads.h"
    erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2685 "beam/erl_threads.h"
#endif
# 2686 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2689 "beam/erl_threads.h"
    ethr_rwmutex_rwlock(&rwmtx->rwmtx);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line);
#endif
# 2693 "beam/erl_threads.h"
#endif
# 2694 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 2703 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 2706 "beam/erl_threads.h"
    ethr_rwmutex_rwunlock(&rwmtx->rwmtx);
#endif
# 2708 "beam/erl_threads.h"
}

#if 0 /* The following rwmtx function names are
	 reserved for potential future use. */ 

/* Try upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE int
erts_rwmtx_trywlock(erts_rwmtx_t *rwmtx)
{
    return 0;
}

/* Upgrade from r-locked state to rw-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wlock(erts_rwmtx_t *rwmtx)
{

}

/* Downgrade from rw-locked state to r-locked state */
ERTS_GLB_INLINE void
erts_rwmtx_wunlock(erts_rwmtx_t *rwmtx)
{

}

#endif
# 2735 "beam/erl_threads.h"

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2746 "beam/erl_threads.h"
    return 0;
#endif
# 2748 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = mtx->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 2760 "beam/erl_threads.h"
    return 0;
#endif
# 2762 "beam/erl_threads.h"
}

/* No atomic ops */

ERTS_GLB_INLINE void
erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    var->sint[0] = val->sint[0];
    var->sint[1] = val->sint[1];
}

ERTS_GLB_INLINE void
erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
    val->sint[0] = var->sint[0];
    val->sint[1] = var->sint[1];
}

ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
					      erts_no_dw_atomic_t *new_val,
					      erts_no_dw_atomic_t *old_val)
{
    if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
	erts_no_dw_atomic_read(var, old_val);
	return 0;
    }
    else {
	erts_no_dw_atomic_set(var, new_val);
	return !0;
    }
}

ERTS_GLB_INLINE void
erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read(erts_no_atomic_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_inc_read(erts_no_atomic_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_dec_read(erts_no_atomic_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic_inc(erts_no_atomic_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic_dec(erts_no_atomic_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_add_read(erts_no_atomic_t *addp, erts_aint_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bor(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_band(erts_no_atomic_t *var, erts_aint_t mask)
{
    erts_aint_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_xchg(erts_no_atomic_t *xchgp, erts_aint_t new)
{
    erts_aint_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
		       erts_aint_t new,
		       erts_aint_t expected)
{
    erts_aint_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint_t
erts_no_atomic_read_bset(erts_no_atomic_t *var,
			 erts_aint_t mask,
			 erts_aint_t set)
{
    erts_aint_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic32 */

ERTS_GLB_INLINE void
erts_no_atomic32_set(erts_no_atomic32_t *var, erts_aint32_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read(erts_no_atomic32_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_inc_read(erts_no_atomic32_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_dec_read(erts_no_atomic32_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_inc(erts_no_atomic32_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic32_dec(erts_no_atomic32_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_add_read(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic32_add(erts_no_atomic32_t *addp, erts_aint32_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bor(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_band(erts_no_atomic32_t *var, erts_aint32_t mask)
{
    erts_aint32_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp, erts_aint32_t new)
{
    erts_aint32_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
			 erts_aint32_t new,
			 erts_aint32_t expected)
{
    erts_aint32_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint32_t
erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
			   erts_aint32_t mask,
			   erts_aint32_t set)
{
    erts_aint32_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* atomic64 */

ERTS_GLB_INLINE void
erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
{
    *var = i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read(erts_no_atomic64_t *var)
{
    return *var;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
{
    return ++(*incp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
{
    return --(*decp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_inc(erts_no_atomic64_t *incp)
{
    ++(*incp);
}

ERTS_GLB_INLINE void
erts_no_atomic64_dec(erts_no_atomic64_t *decp)
{
    --(*decp);
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    return *addp += i;
}

ERTS_GLB_INLINE void
erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
{
    *addp += i;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var |= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
{
    erts_aint64_t old;
    old = *var;
    *var &= mask;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
{
    erts_aint64_t old = *xchgp;
    *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
			 erts_aint64_t new,
			 erts_aint64_t expected)
{
    erts_aint64_t old = *xchgp;
    if (old == expected)
        *xchgp = new;
    return old;
}

ERTS_GLB_INLINE erts_aint64_t
erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
			   erts_aint64_t mask,
			   erts_aint64_t set)
{
    erts_aint64_t old = *var;
    *var &= ~mask;
    *var |= (mask & set);
    return old;
}

/* spinlock */

ERTS_GLB_INLINE void
erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3098 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra);
#endif
# 3101 "beam/erl_threads.h"
#else
# 3102 "beam/erl_threads.h"
    (void)lock;
#endif
# 3104 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra,
			 Uint16 opt)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra);
#endif
# 3117 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra);
#endif
# 3120 "beam/erl_threads.h"
#else
# 3121 "beam/erl_threads.h"
    (void)lock;
#endif
# 3123 "beam/erl_threads.h"
}


ERTS_GLB_INLINE void
erts_spinlock_init(erts_spinlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_spinlock_init(&lock->slck);
    if (res)
	erts_thr_fatal_error(res, "init spinlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK);
#endif
# 3136 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK);
#endif
# 3139 "beam/erl_threads.h"
#else
# 3140 "beam/erl_threads.h"
    (void)lock;
#endif
# 3142 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spinlock_destroy(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3152 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3155 "beam/erl_threads.h"
    res = ethr_spinlock_destroy(&lock->slck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy spinlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3165 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3168 "beam/erl_threads.h"
    (void)lock;
#endif
# 3170 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_spin_unlock(erts_spinlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock(&lock->lc);
#endif
# 3179 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock(&lock->lcnt);
#endif
# 3182 "beam/erl_threads.h"
    ethr_spin_unlock(&lock->slck);
#else
# 3184 "beam/erl_threads.h"
    (void)lock;
#endif
# 3186 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_spin_lock_x(erts_spinlock_t *lock, char *file, unsigned int line)
#else
# 3192 "beam/erl_threads.h"
erts_spin_lock(erts_spinlock_t *lock)
#endif
# 3194 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_x(&lock->lc,file,line);
#else
# 3200 "beam/erl_threads.h"
    erts_lc_lock(&lock->lc);
#endif
# 3202 "beam/erl_threads.h"
#endif
# 3203 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock(&lock->lcnt);
#endif
# 3206 "beam/erl_threads.h"
    ethr_spin_lock(&lock->slck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3210 "beam/erl_threads.h"
#else
# 3211 "beam/erl_threads.h"
    (void)lock;
#endif
# 3213 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_spinlock_is_locked(erts_spinlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = 0;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3225 "beam/erl_threads.h"
    return 0;
#endif
# 3227 "beam/erl_threads.h"
}

/* rwspinlock */

ERTS_GLB_INLINE void
erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra);
#endif
# 3241 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra);
#endif
# 3244 "beam/erl_threads.h"
#else
# 3245 "beam/erl_threads.h"
    (void)lock;
#endif
# 3247 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_init(erts_rwlock_t *lock, char *name)
{
#ifdef USE_THREADS
    int res = ethr_rwlock_init(&lock->rwlck);
    if (res)
	erts_thr_fatal_error(res, "init rwlock");
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK);
#endif
# 3259 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK);
#endif
# 3262 "beam/erl_threads.h"
#else
# 3263 "beam/erl_threads.h"
    (void)lock;
#endif
# 3265 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_rwlock_destroy(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
    int res;
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_destroy_lock(&lock->lc);
#endif
# 3275 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_destroy_lock(&lock->lcnt);
#endif
# 3278 "beam/erl_threads.h"
    res = ethr_rwlock_destroy(&lock->rwlck);
    if (res != 0) {
#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG
	if (res == EBUSY) {
	    char *warn = "Ignoring busy rwlock destroy. "
		"Most likely a bug in pthread implementation.";
	    erts_send_warning_to_logger_str_nogl(warn);
	}
	else
#endif
# 3288 "beam/erl_threads.h"
	    erts_thr_fatal_error(res, "destroy rwlock");
    }
#else
# 3291 "beam/erl_threads.h"
    (void)lock;
#endif
# 3293 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_read_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3302 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3305 "beam/erl_threads.h"
    ethr_read_unlock(&lock->rwlck);
#else
# 3307 "beam/erl_threads.h"
    (void)lock;
#endif
# 3309 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_read_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3315 "beam/erl_threads.h"
erts_read_lock(erts_rwlock_t *lock)
#endif
# 3317 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ,file,line);
#else
# 3323 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ);
#endif
# 3325 "beam/erl_threads.h"
#endif
# 3326 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ);
#endif
# 3329 "beam/erl_threads.h"
    ethr_read_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3333 "beam/erl_threads.h"
#else
# 3334 "beam/erl_threads.h"
    (void)lock;
#endif
# 3336 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_write_unlock(erts_rwlock_t *lock)
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
    erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3345 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3348 "beam/erl_threads.h"
    ethr_write_unlock(&lock->rwlck);
#else
# 3350 "beam/erl_threads.h"
    (void)lock;
#endif
# 3352 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
#ifdef ERTS_ENABLE_LOCK_POSITION
erts_write_lock_x(erts_rwlock_t *lock, char *file, unsigned int line)
#else
# 3358 "beam/erl_threads.h"
erts_write_lock(erts_rwlock_t *lock)
#endif
# 3360 "beam/erl_threads.h"
{
#ifdef USE_THREADS
#ifdef ERTS_ENABLE_LOCK_CHECK
#ifdef ERTS_ENABLE_LOCK_POSITION
    erts_lc_lock_flg_x(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE,file,line);
#else
# 3366 "beam/erl_threads.h"
    erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE);
#endif
# 3368 "beam/erl_threads.h"
#endif
# 3369 "beam/erl_threads.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE);
#endif
# 3372 "beam/erl_threads.h"
    ethr_write_lock(&lock->rwlck);
#ifdef ERTS_ENABLE_LOCK_COUNT
    erts_lcnt_lock_post_x(&lock->lcnt, file, line);
#endif
# 3376 "beam/erl_threads.h"
#else
# 3377 "beam/erl_threads.h"
    (void)lock;
#endif
# 3379 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3391 "beam/erl_threads.h"
    return 0;
#endif
# 3393 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int
erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock)
{
#if defined(USE_THREADS) && defined(ERTS_ENABLE_LOCK_CHECK)
    int res;
    erts_lc_lock_t lc = lock->lc;
    lc.flags = ERTS_LC_FLG_LO_READ|ERTS_LC_FLG_LO_WRITE;
    erts_lc_have_locks(&res, &lc, 1);
    return res;
#else
# 3405 "beam/erl_threads.h"
    return 0;
#endif
# 3407 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_create(erts_tsd_key_t *keyp, char *keyname)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_create(keyp, keyname);
    if (res)
	erts_thr_fatal_error(res, "create thread specific data key");
#endif
# 3417 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_key_delete(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    int res = ethr_tsd_key_delete(key);
    if (res)
	erts_thr_fatal_error(res, "delete thread specific data key");
#endif
# 3427 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_tsd_set(erts_tsd_key_t key, void *value)
{
#ifdef USE_THREADS
    int res = ethr_tsd_set(key, value);
    if (res)
	erts_thr_fatal_error(res, "set thread specific data");
#endif
# 3437 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void *
erts_tsd_get(erts_tsd_key_t key)
{
#ifdef USE_THREADS
    return ethr_tsd_get(key);
#else
# 3445 "beam/erl_threads.h"
    return NULL;
#endif
# 3447 "beam/erl_threads.h"
}

ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void)
{
#ifdef USE_THREADS
    return (erts_tse_t *) ethr_get_ts_event();
#else
# 3454 "beam/erl_threads.h"
    return (erts_tse_t *) NULL;
#endif
# 3456 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_leave_ts_event(ep);
#endif
# 3463 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_prepare_timed(erts_tse_t *ep)
{
#ifdef USE_THREADS
    int res = ethr_event_prepare_timed(&((ethr_ts_event *) ep)->event);
    if (res != 0)
	erts_thr_fatal_error(res, "prepare timed");
#endif
# 3472 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_set(&((ethr_ts_event *) ep)->event);
#endif
# 3479 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep)
{
#ifdef USE_THREADS
    ethr_event_reset(&((ethr_ts_event *) ep)->event);
#endif
# 3486 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return ethr_event_wait(&((ethr_ts_event *) ep)->event);
#else
# 3493 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3495 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount)
{
#ifdef USE_THREADS
    return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount);
#else
# 3502 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3504 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_twait(erts_tse_t *ep, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_twait(&((ethr_ts_event *) ep)->event,
			    (ethr_sint64_t) tmo);
#else
# 3512 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3514 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_stwait(erts_tse_t *ep, int spincount, Sint64 tmo)
{
#ifdef USE_THREADS
    return ethr_event_stwait(&((ethr_ts_event *) ep)->event,
			     spincount,
			     (ethr_sint64_t) tmo);
#else
# 3523 "beam/erl_threads.h"
    return ENOTSUP;
#endif
# 3525 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep)
{
#ifdef USE_THREADS
    return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP;
#else
# 3532 "beam/erl_threads.h"
    return 0;
#endif
# 3534 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no)
{
#ifdef USE_THREADS
    int res = ethr_set_main_thr_status(on, no);
    if (res != 0)
	erts_thr_fatal_error(res, "set thread main status");
#endif
# 3543 "beam/erl_threads.h"
}

ERTS_GLB_INLINE int erts_thr_get_main_status(void)
{
#ifdef USE_THREADS
    int main_status;
    int res = ethr_get_main_thr_status(&main_status);
    if (res != 0)
	erts_thr_fatal_error(res, "get thread main status");
    return main_status;
#else
# 3554 "beam/erl_threads.h"
    return 1;
#endif
# 3556 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void erts_thr_yield(void)
{
#ifdef USE_THREADS
    int res = ETHR_YIELD();
    if (res != 0)
	erts_thr_fatal_error(res, "yield");
#endif    
# 3565 "beam/erl_threads.h"
}


#ifdef ETHR_HAVE_ETHR_SIG_FUNCS

ERTS_GLB_INLINE void
erts_thr_kill(erts_tid_t tid, int sig) {
#ifdef USE_THREADS
  int res = ethr_kill((ethr_tid)tid, sig);
  if (res)
    erts_thr_fatal_error(res, "killing thread");
#endif
# 3577 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigmask(int how, const sigset_t *set, sigset_t *oset)
{
#ifdef USE_THREADS
    int res = ethr_sigmask(how, set, oset);
    if (res)
	erts_thr_fatal_error(res, "get or set signal mask");
#endif
# 3587 "beam/erl_threads.h"
}

ERTS_GLB_INLINE void
erts_thr_sigwait(const sigset_t *set, int *sig)
{
#ifdef USE_THREADS
    int res;
    do {
	res = ethr_sigwait(set, sig);
    } while (res == EINTR);
    if (res)
	erts_thr_fatal_error(res, "to wait for signal");
#endif
# 3600 "beam/erl_threads.h"
}

#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
# 3603 "beam/erl_threads.h"

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 3605 "beam/erl_threads.h"

#endif /* #ifndef ERL_THREAD_H__ */
# 3607 "beam/erl_threads.h"

#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS

/* Deprecated functions to replace */

#undef erts_atomic_init
#undef erts_atomic_set
#undef erts_atomic_read
#undef erts_atomic_inctest
#undef erts_atomic_dectest
#undef erts_atomic_inc
#undef erts_atomic_dec
#undef erts_atomic_addtest
#undef erts_atomic_add
#undef erts_atomic_xchg
#undef erts_atomic_cmpxchg
#undef erts_atomic_bor
#undef erts_atomic_band

#undef erts_atomic32_init
#undef erts_atomic32_set
#undef erts_atomic32_read
#undef erts_atomic32_inctest
#undef erts_atomic32_dectest
#undef erts_atomic32_inc
#undef erts_atomic32_dec
#undef erts_atomic32_addtest
#undef erts_atomic32_add
#undef erts_atomic32_xchg
#undef erts_atomic32_cmpxchg
#undef erts_atomic32_bor
#undef erts_atomic32_band

#endif
# 3641 "beam/erl_threads.h"
# 25 "beam/erl_binary.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "bif.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/erl_binary.h"
# 26 "beam/erl_binary.h"

/*
 * Maximum number of bytes to place in a heap binary.
 */

#define ERL_ONHEAP_BIN_LIMIT 64

/*
 * This structure represents a SUB_BINARY.
 *
 * Note: The last field (orig) is not counted in arityval in the header.
 * This simplifies garbage collection.
 */

typedef struct erl_sub_bin {
    Eterm thing_word;		/* Subtag SUB_BINARY_SUBTAG. */
    Uint size;			/* Binary size in bytes. */
    Uint offs;			/* Offset into original binary. */
    byte bitsize; 
    byte bitoffs; 
    byte is_writable;		/* The underlying binary is writable */
    Eterm orig;			/* Original binary (REFC or HEAP binary). */
} ErlSubBin;

#define ERL_SUB_BIN_SIZE (sizeof(ErlSubBin)/sizeof(Eterm))
#define HEADER_SUB_BIN	_make_header(ERL_SUB_BIN_SIZE-2,_TAG_HEADER_SUB_BIN)

/*
 * This structure represents a HEAP_BINARY.
 */

typedef struct erl_heap_bin {
    Eterm thing_word;		/* Subtag HEAP_BINARY_SUBTAG. */
    Uint size;			/* Binary size in bytes. */
    Eterm data[1];		/* The data in the binary. */
} ErlHeapBin;

#define heap_bin_size(num_bytes)		\
  (sizeof(ErlHeapBin)/sizeof(Eterm) - 1 +	\
   ((num_bytes)+sizeof(Eterm)-1)/sizeof(Eterm))

#define header_heap_bin(num_bytes) \
  _make_header(heap_bin_size(num_bytes)-1,_TAG_HEADER_HEAP_BIN)

/*
 * Get the size in bytes of any type of binary.
 */

#define binary_size(Bin) (binary_val(Bin)[1])
#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1])

#define binary_bitsize(Bin)			\
  ((*binary_val(Bin) == HEADER_SUB_BIN) ?	\
   ((ErlSubBin *) binary_val(Bin))->bitsize:	\
   0)

#define binary_bitoffset(Bin)			\
  ((*binary_val(Bin) == HEADER_SUB_BIN) ?	\
   ((ErlSubBin *) binary_val(Bin))->bitoffs:	\
   0)

/*
 * Get the pointer to the actual data bytes in a binary.
 * Works for any type of binary. Always use binary_bytes() if
 * you know that the binary cannot be a sub binary.
 *
 * Bin: input variable (Eterm)
 * Bytep: output variable (byte *)
 * Bitoffs: output variable (Uint)
 * Bitsize: output variable (Uint)
 */

#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \
     ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL)

#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr)    \
do {									\
    Eterm* _real_bin = binary_val_rel(Bin,BasePtr);			\
    Uint _offs = 0;							\
    Bitoffs = Bitsize = 0;						\
    if (*_real_bin == HEADER_SUB_BIN) {					\
	ErlSubBin* _sb = (ErlSubBin *) _real_bin;			\
	_offs = _sb->offs;						\
        Bitoffs = _sb->bitoffs;						\
        Bitsize = _sb->bitsize;						\
	_real_bin = binary_val_rel(_sb->orig,BasePtr);			\
    }									\
    if (*_real_bin == HEADER_PROC_BIN) {				\
	Bytep = ((ProcBin *) _real_bin)->bytes + _offs;			\
    } else {								\
	Bytep = (byte *)(&(((ErlHeapBin *) _real_bin)->data)) + _offs;	\
    }									\
} while (0)

/*
 * Get the real binary from any binary type, where "real" means
 * a REFC or HEAP binary. Also get the byte and bit offset into the
 * real binary. Useful if you want to build a SUB binary from
 * any binary.
 *
 * Bin: Input variable (Eterm)
 * RealBin: Output variable (Eterm)
 * ByteOffset: Output variable (Uint)
 * BitOffset: Offset in bits (Uint)
 * BitSize: Extra bit size (Uint)
 */

#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \
     ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL)

#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \
  do {									\
    ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr);	        \
    if (_sb->thing_word == HEADER_SUB_BIN) {				\
      RealBin = _sb->orig;						\
      ByteOffset = _sb->offs;						\
      BitOffset = _sb->bitoffs;						\
      BitSize = _sb->bitsize;						\
    } else {								\
      RealBin = Bin;							\
      ByteOffset = BitOffset = BitSize = 0;				\
    }									\
  } while (0)

/*
 * Get a pointer to the binary bytes, for a heap or refc binary
 * (NOT sub binary).
 */
#define binary_bytes(Bin)						\
  (*binary_val(Bin) == HEADER_PROC_BIN ?				\
   ((ProcBin *) binary_val(Bin))->bytes :				\
   (ASSERT(thing_subtag(*binary_val(Bin)) == HEAP_BINARY_SUBTAG),	\
   (byte *)(&(((ErlHeapBin *) binary_val(Bin))->data))))

void erts_init_binary(void);

byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra);
/* Used by unicode module */
Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs);

/*
 * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1
 */

BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg, Export *bif);
BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple);
BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen);


#if defined(__i386__) || !defined(__GNUC__)
/*
 * Doubles aren't required to be 8-byte aligned on intel x86.
 * (if not gnuc we don't know if __i386__ is defined on x86;
 *  therefore, assume intel x86...)
 */
#  define ERTS_BIN_ALIGNMENT_MASK ((Uint) 3)
#else
# 183 "beam/erl_binary.h"
#  define ERTS_BIN_ALIGNMENT_MASK ((Uint) 7)
#endif
# 185 "beam/erl_binary.h"

#define ERTS_CHK_BIN_ALIGNMENT(B) \
  do { ASSERT(!(B) || (((UWord) &((Binary *)(B))->orig_bytes[0]) & ERTS_BIN_ALIGNMENT_MASK) == ((UWord) 0)); } while(0)

ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr);
ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf);
ERTS_GLB_INLINE void erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t);
ERTS_GLB_INLINE Binary *erts_bin_drv_alloc_fnf(Uint size);
ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size);
ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size);
ERTS_GLB_INLINE Binary *erts_bin_realloc_fnf(Binary *bp, Uint size);
ERTS_GLB_INLINE Binary *erts_bin_realloc(Binary *bp, Uint size);
ERTS_GLB_INLINE void erts_bin_free(Binary *bp);
ERTS_GLB_INLINE Binary *erts_create_magic_binary_x(Uint size,
                                                  void (*destructor)(Binary *),
                                                  int unaligned);
ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size,
						 void (*destructor)(Binary *));

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

#if 0 /* expanded by -frewrite-includes */
#include <stddef.h> /* offsetof */
#endif /* expanded by -frewrite-includes */
# 206 "beam/erl_binary.h"
# 207 "beam/erl_binary.h"

ERTS_GLB_INLINE byte*
erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr)
{
    return erts_get_aligned_binary_bytes_extra(bin, base_ptr, ERTS_ALC_T_TMP, 0);
}

ERTS_GLB_INLINE void
erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t allocator)
{
    if (buf) {
	erts_free(allocator, (void *) buf);
    }
}

ERTS_GLB_INLINE void
erts_free_aligned_binary_bytes(byte* buf)
{
    erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP);
}

/* Explicit extra bytes allocated to counter buggy drivers.
** These extra bytes where earlier (< R13B04) added by an alignment-bug
** in this code. Do we dare remove this in some major release (R14?) maybe?
*/
#if defined(DEBUG) || defined(VALGRIND)
#  define CHICKEN_PAD 0
#else
# 235 "beam/erl_binary.h"
#  define CHICKEN_PAD (sizeof(void*) - 1)
#endif
# 237 "beam/erl_binary.h"

/* Caller must initialize 'refc'
*/
ERTS_GLB_INLINE Binary *
erts_bin_drv_alloc_fnf(Uint size)
{
    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
    Binary *res;

    if (bsize < size) /* overflow */
	return NULL;
    res = erts_alloc_fnf(ERTS_ALC_T_DRV_BINARY, bsize);
    ERTS_CHK_BIN_ALIGNMENT(res);
    if (res) {
	res->orig_size = size;
	res->flags = BIN_FLAG_DRV;
    }
    return res;
}

/* Caller must initialize 'refc'
*/
ERTS_GLB_INLINE Binary *
erts_bin_drv_alloc(Uint size)
{
    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
    Binary *res;

    if (bsize < size) /* overflow */
	erts_alloc_enomem(ERTS_ALC_T_DRV_BINARY, size);
    res = erts_alloc(ERTS_ALC_T_DRV_BINARY, bsize);
    ERTS_CHK_BIN_ALIGNMENT(res);
    res->orig_size = size;
    res->flags = BIN_FLAG_DRV;
    return res;
}


/* Caller must initialize 'refc'
*/
ERTS_GLB_INLINE Binary *
erts_bin_nrml_alloc(Uint size)
{
    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
    Binary *res;

    if (bsize < size) /* overflow */
	erts_alloc_enomem(ERTS_ALC_T_BINARY, size);
    res = erts_alloc(ERTS_ALC_T_BINARY, bsize);
    ERTS_CHK_BIN_ALIGNMENT(res);
    res->orig_size = size;
    res->flags = 0;
    return res;
}

ERTS_GLB_INLINE Binary *
erts_bin_realloc_fnf(Binary *bp, Uint size)
{
    Binary *nbp;
    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
	                                            : ERTS_ALC_T_BINARY;
    ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
    if (bsize < size) /* overflow */
	return NULL;
    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
    ERTS_CHK_BIN_ALIGNMENT(nbp);
    if (nbp)
	nbp->orig_size = size;
    return nbp;
}

ERTS_GLB_INLINE Binary *
erts_bin_realloc(Binary *bp, Uint size)
{
    Binary *nbp;
    Uint bsize = ERTS_SIZEOF_Binary(size) + CHICKEN_PAD;
    ErtsAlcType_t type = (bp->flags & BIN_FLAG_DRV) ? ERTS_ALC_T_DRV_BINARY
	                                            : ERTS_ALC_T_BINARY;
    ASSERT((bp->flags & BIN_FLAG_MAGIC) == 0);
    if (bsize < size) /* overflow */
	erts_realloc_enomem(type, bp, size);
    nbp = erts_realloc_fnf(type, (void *) bp, bsize);
    if (!nbp)
	erts_realloc_enomem(type, bp, bsize);
    ERTS_CHK_BIN_ALIGNMENT(nbp);
    nbp->orig_size = size;
    return nbp;
}

ERTS_GLB_INLINE void
erts_bin_free(Binary *bp)
{
    if (bp->flags & BIN_FLAG_MAGIC)
	ERTS_MAGIC_BIN_DESTRUCTOR(bp)(bp);
    if (bp->flags & BIN_FLAG_DRV)
	erts_free(ERTS_ALC_T_DRV_BINARY, (void *) bp);
    else
	erts_free(ERTS_ALC_T_BINARY, (void *) bp);
}

ERTS_GLB_INLINE Binary *
erts_create_magic_binary_x(Uint size, void (*destructor)(Binary *),
                           int unaligned)
{
    Uint bsize = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_SIZE(size)
                           : ERTS_MAGIC_BIN_SIZE(size);
    Binary* bptr = erts_alloc_fnf(ERTS_ALC_T_BINARY, bsize);
    ASSERT(bsize > size);
    if (!bptr)
	erts_alloc_n_enomem(ERTS_ALC_T2N(ERTS_ALC_T_BINARY), bsize);
    ERTS_CHK_BIN_ALIGNMENT(bptr);
    bptr->flags = BIN_FLAG_MAGIC;
    bptr->orig_size = unaligned ? ERTS_MAGIC_BIN_UNALIGNED_ORIG_SIZE(size)
                                : ERTS_MAGIC_BIN_ORIG_SIZE(size);
    erts_refc_init(&bptr->refc, 0);
    ERTS_MAGIC_BIN_DESTRUCTOR(bptr) = destructor;
    return bptr;
}

ERTS_GLB_INLINE Binary *
erts_create_magic_binary(Uint size, void (*destructor)(Binary *))
{
    return erts_create_magic_binary_x(size, destructor, 0);
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 364 "beam/erl_binary.h"

#endif /* !__ERL_BINARY_H */
# 366 "beam/erl_binary.h"
# 35 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_map.h"
#endif /* expanded by -frewrite-includes */
# 35 "beam/beam_emu.c"
# 1 "beam/erl_map.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2014. All Rights Reserved.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 * %CopyrightEnd%
 */


#ifndef __ERL_MAP_H__
#define __ERL_MAP_H__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/erl_map.h"
# 26 "beam/erl_map.h"

/* instrinsic wrappers */
#if ERTS_AT_LEAST_GCC_VSN__(3, 4, 0)
#define hashmap_clz(x)       ((Uint32) __builtin_clz((unsigned int)(x)))
#define hashmap_bitcount(x)  ((Uint32) __builtin_popcount((unsigned int) (x)))
#else
# 32 "beam/erl_map.h"
Uint32 hashmap_clz(Uint32 x);
Uint32 hashmap_bitcount(Uint32 x);
#endif
# 35 "beam/erl_map.h"

/* MAP */

typedef struct flatmap_s {
    Eterm thing_word;
    Uint  size;
    Eterm keys;      /* tuple */
} flatmap_t;
/* map node
 *
 * -----------
 * Eterm   THING	
 * Uint    size
 * Eterm   Keys -> {K1, K2, K3, ..., Kn} where n = size
 * ----
 * Eterm   V1
 * ...
 * Eterm   Vn, where n = size
 * -----------
 */

/* the head-node is a bitmap or array with an untagged size */


#define hashmap_size(x)               (((hashmap_head_t*) hashmap_val(x))->size)
#define hashmap_size_rel(RTERM, BASE) hashmap_size(rterm2wterm(RTERM, BASE))
#define hashmap_make_hash(Key)        make_internal_hash(Key)

#define hashmap_restore_hash(Heap,Lvl,Key) \
    (((Lvl) < 8) ? hashmap_make_hash(Key) >> (4*(Lvl)) : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), (Key))) >> (4*((Lvl) & 7)))
#define hashmap_shift_hash(Heap,Hx,Lvl,Key) \
    (((++(Lvl)) & 7) ? (Hx) >> 4 : hashmap_make_hash(CONS(Heap, make_small((Lvl)>>3), Key)))


/* erl_term.h stuff */
#define flatmap_get_values(x)        (((Eterm *)(x)) + 3)
#define flatmap_get_keys(x)          (((Eterm *)tuple_val(((flatmap_t *)(x))->keys)) + 1)
#define flatmap_get_size(x)          (((flatmap_t*)(x))->size)

#ifdef DEBUG
#define MAP_SMALL_MAP_LIMIT    (3)
#else
# 77 "beam/erl_map.h"
#define MAP_SMALL_MAP_LIMIT    (32)
#endif
# 79 "beam/erl_map.h"

struct ErtsWStack_;
struct ErtsEStack_;

Eterm  erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map);
int    erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res);
int    erts_maps_remove(Process *p, Eterm key, Eterm map, Eterm *res);

Eterm  erts_hashmap_insert(Process *p, Uint32 hx, Eterm key, Eterm value,
			   Eterm node, int is_update);
int    erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz,
			        Uint *upsz, struct ErtsEStack_ *sp, int is_update);
Eterm  erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value,
			      Uint *upsz, struct ErtsEStack_ *sp);

int    erts_validate_and_sort_flatmap(flatmap_t* map);
void   hashmap_iterator_init(struct ErtsWStack_* s, Eterm node, int reverse);
Eterm* hashmap_iterator_next(struct ErtsWStack_* s);
Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys);

#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \
    erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE);

Eterm  erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
					 Eterm k, Eterm v);

const Eterm *
#if HALFWORD_HEAP
erts_maps_get_rel(Eterm key, Eterm map, Eterm *map_base);
#  define erts_maps_get(A, B) erts_maps_get_rel(A, B, NULL)
#else
# 112 "beam/erl_map.h"
erts_maps_get(Eterm key, Eterm map);
#  define erts_maps_get_rel(A, B, B_BASE) erts_maps_get(A, B)
#endif
# 115 "beam/erl_map.h"

const Eterm *
#if HALFWORD_HEAP
erts_hashmap_get_rel(Uint32 hx, Eterm key, Eterm node, Eterm *map_base);
#  define erts_hashmap_get(Hx, K, M) erts_hashmap_get_rel(Hx, K, M, NULL)
#else
# 121 "beam/erl_map.h"
erts_hashmap_get(Uint32 hx, Eterm key, Eterm map);
#  define erts_hashmap_get_rel(Hx, K, M, M_BASE) erts_hashmap_get(Hx, K, M)
#endif
# 124 "beam/erl_map.h"

/* hamt nodes v2.0
 *
 * node :: leaf | array | bitmap
 * head
 */
typedef struct hashmap_head_s {
    Eterm thing_word;
    Uint size;
    Eterm items[1];
} hashmap_head_t;

/* thing_word tagscheme
 * Need two bits for map subtags
 *
 * Original HEADER representation:
 *
 *     aaaaaaaaaaaaaaaa aaaaaaaaaatttt00       arity:26, tag:4
 *
 * For maps we have:
 *
 *     vvvvvvvvvvvvvvvv aaaaaaaamm111100       val:16, arity:8, mtype:2
 *
 * unsure about trailing zeros
 *
 * map-tag:
 *     00 - flat map tag (non-hamt) -> val:16 = #items
 *     01 - map-node bitmap tag     -> val:16 = bitmap
 *     10 - map-head (array-node)   -> val:16 = 0xffff
 *     11 - map-head (bitmap-node)  -> val:16 = bitmap
 */

/* erl_map.h stuff */

#define is_hashmap_header_head(x) ((MAP_HEADER_TYPE(x) & (0x2)))

#define MAKE_MAP_HEADER(Type,Arity,Val) \
    (_make_header(((((Uint16)(Val)) << MAP_HEADER_ARITY_SZ) | (Arity)) << MAP_HEADER_TAG_SZ | (Type) , _TAG_HEADER_MAP))

#define MAP_HEADER_FLATMAP \
    MAKE_MAP_HEADER(MAP_HEADER_TAG_FLATMAP_HEAD,0x1,0x0)

#define MAP_HEADER_HAMT_HEAD_ARRAY \
    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_ARRAY,0x1,0xffff)

#define MAP_HEADER_HAMT_HEAD_BITMAP(Bmp) \
    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_HEAD_BITMAP,0x1,Bmp)

#define MAP_HEADER_HAMT_NODE_BITMAP(Bmp) \
    MAKE_MAP_HEADER(MAP_HEADER_TAG_HAMT_NODE_BITMAP,0x0,Bmp)

#define MAP_HEADER_FLATMAP_SZ  (sizeof(flatmap_t) / sizeof(Eterm))

#define HAMT_NODE_ARRAY_SZ      (17)
#define HAMT_HEAD_ARRAY_SZ      (18)
#define HAMT_NODE_BITMAP_SZ(n)  (1 + n)
#define HAMT_HEAD_BITMAP_SZ(n)  (2 + n)

/* 2 bits maps tag + 4 bits subtag + 2 ignore bits */
#define _HEADER_MAP_SUBTAG_MASK       (0xfc)
/* 1 bit map tag + 1 ignore bit + 4 bits subtag + 2 ignore bits */
#define _HEADER_MAP_HASHMAP_HEAD_MASK (0xbc)

#define HAMT_SUBTAG_NODE_BITMAP  ((MAP_HEADER_TAG_HAMT_NODE_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
#define HAMT_SUBTAG_HEAD_ARRAY   ((MAP_HEADER_TAG_HAMT_HEAD_ARRAY  << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
#define HAMT_SUBTAG_HEAD_BITMAP  ((MAP_HEADER_TAG_HAMT_HEAD_BITMAP << _HEADER_ARITY_OFFS) | MAP_SUBTAG)
#define HAMT_SUBTAG_HEAD_FLATMAP ((MAP_HEADER_TAG_FLATMAP_HEAD << _HEADER_ARITY_OFFS) | MAP_SUBTAG)

#define hashmap_index(hash)      (((Uint32)hash) & 0xf)

/* hashmap heap size:
   [one cons cell + one list term in parent node] per key
   [one header + one boxed term in parent node] per inner node
   [one header + one size word] for root node
   Observed average number of nodes per key is about 0.35.
*/
#define HASHMAP_WORDS_PER_KEY 3
#define HASHMAP_WORDS_PER_NODE 2
#ifdef DEBUG
#  define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \
    (HASHMAP_WORDS_PER_NODE * (KEYS) * 3/10)   /* slightly under estimated */
#else
# 206 "beam/erl_map.h"
#  define HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS) \
    (HASHMAP_WORDS_PER_NODE * (KEYS) * 4/10)   /* slightly over estimated */
#endif
# 209 "beam/erl_map.h"
#define HASHMAP_ESTIMATED_HEAP_SIZE(KEYS) \
        ((KEYS)*HASHMAP_WORDS_PER_KEY + HASHMAP_ESTIMATED_TOT_NODE_SIZE(KEYS))
#endif
# 212 "beam/erl_map.h"
# 36 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 36 "beam/beam_emu.c"
# 37 "beam/beam_emu.c"
#if 0 /* expanded by -frewrite-includes */
#include "dist.h"
#endif /* expanded by -frewrite-includes */
# 37 "beam/beam_emu.c"
# 1 "beam/dist.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __DIST_H__
#define __DIST_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 24 "beam/dist.h"
# 1 "beam/erl_process.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2014. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __PROCESS_H__
#define __PROCESS_H__

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS
#if (defined(ERL_PROCESS_C__) \
     || defined(ERL_PORT_TASK_C__) \
     || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
#define ERTS_INCLUDE_SCHEDULER_INTERNALS
#endif
# 31 "beam/erl_process.h"

/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */

#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
#  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#endif
# 37 "beam/erl_process.h"

typedef struct process Process;

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/erl_process.h"
# 41 "beam/erl_process.h"

#define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h" /* Only pull out important types... */
#endif /* expanded by -frewrite-includes */
# 43 "beam/erl_process.h"
# 44 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__

#define ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_port.h"
#endif /* expanded by -frewrite-includes */
# 47 "beam/erl_process.h"
# 48 "beam/erl_process.h"
#undef ERL_PORT_GET_PORT_TYPE_ONLY__
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 49 "beam/erl_process.h"
# 50 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_smp.h"
#endif /* expanded by -frewrite-includes */
# 50 "beam/erl_process.h"
# 51 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_message.h"
#endif /* expanded by -frewrite-includes */
# 51 "beam/erl_process.h"
# 52 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_dict.h"
#endif /* expanded by -frewrite-includes */
# 52 "beam/erl_process.h"
# 53 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_container_utils.h"
#endif /* expanded by -frewrite-includes */
# 53 "beam/erl_process.h"
# 54 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 54 "beam/erl_process.h"
# 55 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_monitors.h"
#endif /* expanded by -frewrite-includes */
# 55 "beam/erl_process.h"
# 56 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_hl_timer.h"
#endif /* expanded by -frewrite-includes */
# 56 "beam/erl_process.h"
# 57 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_time.h"
#endif /* expanded by -frewrite-includes */
# 57 "beam/erl_process.h"
# 58 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_atom_table.h"
#endif /* expanded by -frewrite-includes */
# 58 "beam/erl_process.h"
# 59 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "external.h"
#endif /* expanded by -frewrite-includes */
# 59 "beam/erl_process.h"
# 60 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_mseg.h"
#endif /* expanded by -frewrite-includes */
# 60 "beam/erl_process.h"
# 61 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_async.h"
#endif /* expanded by -frewrite-includes */
# 61 "beam/erl_process.h"
# 62 "beam/erl_process.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 62 "beam/erl_process.h"
# 63 "beam/erl_process.h"
#define ERTS_ONLY_INCLUDE_TRACE_FLAGS
#if 0 /* expanded by -frewrite-includes */
#include "erl_trace.h"
#endif /* expanded by -frewrite-includes */
# 64 "beam/erl_process.h"
# 65 "beam/erl_process.h"
#undef ERTS_ONLY_INCLUDE_TRACE_FLAGS

#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_process.h"
#endif /* expanded by -frewrite-includes */
# 68 "beam/erl_process.h"
# 69 "beam/erl_process.h"
#endif
# 70 "beam/erl_process.h"

#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 73 "beam/erl_process.h"
# 74 "beam/erl_process.h"
#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY

struct ErtsNodesMonitor_;

#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0

#define ERTS_MAX_NO_OF_SCHEDULERS 1024
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
#endif
# 86 "beam/erl_process.h"

#define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)

#define ERTS_HEAP_ALLOC(Type, Size)					\
     erts_alloc((Type), (Size))

#define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
     erts_realloc((Type), (Ptr), (NewSize))

#define ERTS_HEAP_FREE(Type, Ptr, Size)					\
     erts_free((Type), (Ptr))

#define INITIAL_MOD 0
#define INITIAL_FUN 1
#define INITIAL_ARI 2

#if 0 /* expanded by -frewrite-includes */
#include "export.h"
#endif /* expanded by -frewrite-includes */
# 102 "beam/erl_process.h"
# 103 "beam/erl_process.h"

struct saved_calls {
   int len;
   int n;
   int cur;
   Export *ct[1];
};

extern Export exp_send, exp_receive, exp_timeout;
extern int erts_eager_check_io;
extern int erts_sched_compact_load;
extern int erts_sched_balance_util;
extern Uint erts_no_schedulers;
#ifdef ERTS_DIRTY_SCHEDULERS
extern Uint erts_no_dirty_cpu_schedulers;
extern Uint erts_no_dirty_io_schedulers;
#endif
# 120 "beam/erl_process.h"
extern Uint erts_no_run_queues;
extern int erts_sched_thread_suggested_stack_size;
#define ERTS_SCHED_THREAD_MIN_STACK_SIZE 4	/* Kilo words */
#define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */

#ifdef ERTS_SMP
#if 0 /* expanded by -frewrite-includes */
#include "erl_bits.h"
#endif /* expanded by -frewrite-includes */
# 126 "beam/erl_process.h"
# 127 "beam/erl_process.h"
#endif
# 128 "beam/erl_process.h"

/* process priorities */
#define PRIORITY_MAX          0
#define PRIORITY_HIGH         1
#define PRIORITY_NORMAL       2
#define PRIORITY_LOW          3
#define ERTS_NO_PROC_PRIO_LEVELS      4
#define ERTS_NO_PROC_PRIO_QUEUES      3

#define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
#define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)

#define ERTS_RUNQ_FLGS_PROCS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_QMASK \
  ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)

#define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
  ERTS_NO_PRIO_LEVELS
#define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
  (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
  (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
#define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
#define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
  (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)

#define ERTS_RUNQ_FLG_BASE2 \
  (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)

#define ERTS_RUNQ_FLG_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
#define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
#define ERTS_RUNQ_FLG_SUSPENDED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
#define ERTS_RUNQ_FLG_CHK_CPU_BIND \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
#define ERTS_RUNQ_FLG_INACTIVE \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
#define ERTS_RUNQ_FLG_NONEMPTY \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
#define ERTS_RUNQ_FLG_PROTECTED \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
#define ERTS_RUNQ_FLG_EXEC \
  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))

#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)

#define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
  (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
   | ERTS_RUNQ_FLGS_EVACUATE_QMASK)

#define ERTS_RUNQ_FLGS_MIGRATION_INFO \
  (ERTS_RUNQ_FLG_INACTIVE \
   | ERTS_RUNQ_FLG_OUT_OF_WORK \
   | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)

#define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))

#define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
  (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
#define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
#define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
  ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))

#define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
    erts_smp_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
#define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_bor_relb(&(RQ)->flags,		\
					      (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bor_nob(&(RQ)->flags,		\
					     (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags,		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))
#define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_relb(&(RQ)->flags,		\
					       (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
    ((Uint32) erts_smp_atomic32_read_band_nob(&(RQ)->flags,		\
					      (erts_aint32_t) ~(FLGS)))
#define ERTS_RUNQ_FLGS_GET(RQ)						\
    ((Uint32) erts_smp_atomic32_read_acqb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_nob(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
    ((Uint32) erts_smp_atomic32_read_mb(&(RQ)->flags))
#define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
    ((Uint32) erts_smp_atomic32_read_bset_relb(&(RQ)->flags, 		\
					       (erts_aint32_t) (MSK),	\
					       (erts_aint32_t) (FLGS)))

typedef enum {
    ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
    ERTS_SCHDLR_SSPND_DONE,
    ERTS_SCHDLR_SSPND_YIELD_RESTART,
    ERTS_SCHDLR_SSPND_YIELD_DONE,
    ERTS_SCHDLR_SSPND_EINVAL
} ErtsSchedSuspendResult;

typedef enum {
    ERTS_MIGRATE_SUCCESS,
    ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
    ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
    ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
} ErtsMigrateResult;

#define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
#define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
#define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
#define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
#define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)

#define ERTS_SSI_FLGS_MAX                                       5

#define ERTS_SSI_FLGS_SLEEP_TYPE			\
 (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)

#define ERTS_SSI_FLGS_SLEEP				\
 (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)

#define ERTS_SSI_FLGS_ALL				\
 (ERTS_SSI_FLGS_SLEEP					\
  | ERTS_SSI_FLG_WAITING				\
  | ERTS_SSI_FLG_SUSPENDED)

/*
 * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
 * order relative eachother. Most frequent at lowest at lowest
 * index.
 *
 * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
 * highest index...
 *
 * Remember to update description in erts_pre_init_process()
 * when adding new flags...
 */

typedef enum {
    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
    ERTS_SSI_AUX_WORK_DD_IX,
    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
    ERTS_SSI_AUX_WORK_MISC_IX,
    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
    ERTS_SSI_AUX_WORK_SET_TMO_IX,
    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */

    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
} ErtsSsiAuxWorkFlagIndex;

#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
#define ERTS_SSI_AUX_WORK_DD \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
#define ERTS_SSI_AUX_WORK_MISC \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
#define ERTS_SSI_AUX_WORK_SET_TMO \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
#define ERTS_SSI_AUX_WORK_REAP_PORTS \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)

typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef struct {
    erts_smp_spinlock_t lock;
    ErtsSchedulerSleepInfo *list;
} ErtsSchedulerSleepList;
#endif
# 359 "beam/erl_process.h"

struct ErtsSchedulerSleepInfo_ {
#ifdef ERTS_SMP
    ErtsSchedulerSleepInfo *next;
    ErtsSchedulerSleepInfo *prev;
    erts_smp_atomic32_t flags;
    erts_tse_t *event;
#endif
# 367 "beam/erl_process.h"
    erts_atomic32_t aux_work;
};

/* times to reschedule low prio process before running */
#define RESCHEDULE_LOW        8

#define ERTS_MAX_MISC_OPS 5

#define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
#define ERTS_FULL_REDS_HISTORY_SIZE \
   ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)

typedef struct ErtsProcList_ ErtsProcList;
struct ErtsProcList_ {
    Eterm pid;
    Uint64 started_interval;
    ErtsProcList* next;
    ErtsProcList* prev;
};

typedef struct ErtsMiscOpList_ ErtsMiscOpList;
struct ErtsMiscOpList_ {
    ErtsMiscOpList *next;
    void (*func)(void *arg);
    void *arg;
};

typedef struct {
    Process* first;
    Process* last;
} ErtsRunPrioQueue;

typedef struct ErtsSchedulerData_ ErtsSchedulerData;

typedef struct ErtsRunQueue_ ErtsRunQueue;

typedef struct {
    erts_smp_atomic32_t len;
    erts_aint32_t max_len;
    int reds;
} ErtsRunQueueInfo;


#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
#  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
#  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
#endif
# 414 "beam/erl_process.h"

#ifdef ERTS_SMP

#undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
#define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT

typedef erts_atomic64_t ErtsAtomicSchedTime;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
typedef struct {
    ErtsAtomicSchedTime last;
    struct {
	Uint64 short_interval;
	Uint64 long_interval;
    } worktime;
    int is_working;
} ErtsRunQueueSchedUtil;
#endif
# 432 "beam/erl_process.h"

typedef struct {
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    int sched_util;
#endif
# 437 "beam/erl_process.h"
    Uint32 flags;
    ErtsRunQueue *misc_evac_runq;
    struct {
	struct {
	    int this;
	    int other;
	} limit;
	ErtsRunQueue *runq;
	Uint32 flags;
    } prio[ERTS_NO_PRIO_LEVELS];
} ErtsMigrationPath;

typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;

struct ErtsMigrationPaths_ {
    void *block;
    ErtsMigrationPaths *next;
    ErtsThrPrgrVal thr_prgr;
    ErtsMigrationPath mpath[1];
};

#endif /* ERTS_SMP */
# 459 "beam/erl_process.h"

struct ErtsRunQueue_ {
    int ix;

    erts_smp_mtx_t mtx;
    erts_smp_cnd_t cnd;

#ifdef ERTS_DIRTY_SCHEDULERS
#ifdef ERTS_SMP
    ErtsSchedulerSleepList sleepers;
#endif
# 470 "beam/erl_process.h"
#endif
# 471 "beam/erl_process.h"

    ErtsSchedulerData *scheduler;
    int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
    int woken;
    erts_smp_atomic32_t flags;
    int check_balance_reds;
    int full_reds_history_sum;
    int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
    int out_of_work_count;
    erts_aint32_t max_len;
    erts_smp_atomic32_t len;
    int wakeup_other;
    int wakeup_other_reds;
    int halt_in_progress;

    struct {
	ErtsProcList *pending_exiters;
	Uint context_switches;
	Uint reductions;

	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];

	/* We use the same prio queue for low and
	   normal prio processes */
	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
    } procs;

    struct {
	ErtsMiscOpList *start;
	ErtsMiscOpList *end;
	erts_smp_atomic_t evac_runq;
    } misc;

    struct {
	ErtsRunQueueInfo info;
	Port *start;
	Port *end;
    } ports;
#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
    ErtsRunQueueSchedUtil sched_util;
#endif
# 512 "beam/erl_process.h"
};

#ifdef ERTS_SMP
extern long erts_runq_supervision_interval;
#endif
# 517 "beam/erl_process.h"

typedef union {
    ErtsRunQueue runq;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
} ErtsAlignedRunQueue;

extern ErtsAlignedRunQueue *erts_aligned_run_queues;

#define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
do {								\
    (RQ)->procs.reductions += (AREDS);				\
    (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (AREDS);				\
    (SD)->check_time_reds += (AREDS);				\
} while (0)

#define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
do {								\
    (RQ)->ports.info.reds += (REDS);				\
    (RQ)->check_balance_reds -= (REDS);				\
    (RQ)->wakeup_other_reds += (REDS);				\
    (SD)->check_time_reds += (REDS);				\
} while (0)

typedef struct {
    int need; /* "+sbu true" or scheduler_wall_time enabled */
    int enabled;
    Uint64 start;
    struct {
	Uint64 total;
	Uint64 start;
	int currently;
    } working;
} ErtsSchedWallTime;

typedef struct {
    int sched;
    erts_aint32_t aux_work;
} ErtsDelayedAuxWorkWakeupJob;

typedef struct {
    int sched_id;
    ErtsSchedulerData *esdp;
    ErtsSchedulerSleepInfo *ssi;
#ifdef ERTS_SMP
    ErtsThrPrgrVal current_thr_prgr;
    ErtsThrPrgrVal latest_wakeup;
#endif
# 566 "beam/erl_process.h"
    struct {
	int ix;
#ifdef ERTS_SMP
	ErtsThrPrgrVal thr_prgr;
#endif
# 571 "beam/erl_process.h"
    } misc;
#ifdef ERTS_SMP
    struct {
	ErtsThrPrgrVal thr_prgr;
    } dd;
    struct {
	ErtsThrPrgrVal thr_prgr;
    } cncld_tmrs;
    struct {
	ErtsThrPrgrVal thr_prgr;
	UWord size;
	ErtsThrPrgrLaterOp *first;
	ErtsThrPrgrLaterOp *last;
    } later_op;
#endif
# 586 "beam/erl_process.h"
#ifdef ERTS_USE_ASYNC_READY_Q
    struct {
#ifdef ERTS_SMP
	int need_thr_prgr;
	ErtsThrPrgrVal thr_prgr;
#endif
# 592 "beam/erl_process.h"
	void *queue;
    } async_ready;
#endif
# 595 "beam/erl_process.h"
#ifdef ERTS_SMP
    struct {
	Uint64 next;
	int *sched2jix;
	int jix;
	ErtsDelayedAuxWorkWakeupJob *job;
    } delayed_wakeup;
#endif
# 603 "beam/erl_process.h"
    struct {
	struct {
	    erts_aint32_t flags;
	    void (*callback)(void *);
	    void *arg;
	} wait_completed;
    } debug;
} ErtsAuxWorkData;

#ifdef ERTS_DIRTY_SCHEDULERS
typedef enum {
    ERTS_DIRTY_CPU_SCHEDULER,
    ERTS_DIRTY_IO_SCHEDULER
} ErtsDirtySchedulerType;

typedef union {
    struct {
	ErtsDirtySchedulerType type: 1;
	Uint num: sizeof(Uint)*8 - 1;
    } s;
    Uint no;
} ErtsDirtySchedId;
#endif
# 626 "beam/erl_process.h"

struct ErtsSchedulerData_ {
    /*
     * Keep X registers first (so we get as many low
     * numbered registers as possible in the same cache
     * line).
     */
    Eterm* x_reg_array;		/* X registers */
    FloatDef* f_reg_array;	/* Floating point registers. */

    ErtsTimerWheel *timer_wheel;
    ErtsNextTimeoutRef next_tmo_ref;
    ErtsHLTimerService *timer_service;
#ifdef ERTS_SMP
    ethr_tid tid;		/* Thread id */
    struct erl_bits_state erl_bits_state; /* erl_bits.c state */
    void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
    Process *free_process;
    ErtsThrPrgrData thr_progress_data;
#endif
# 646 "beam/erl_process.h"
#if !HEAP_ON_C_STACK
    Eterm tmp_heap[TMP_HEAP_SIZE];
    int num_tmp_heap_used;
    Eterm beam_emu_tmp_heap[BEAM_EMU_TMP_HEAP_SIZE];
    Eterm erl_arith_tmp_heap[ERL_ARITH_TMP_HEAP_SIZE];
#endif
# 652 "beam/erl_process.h"
    ErtsSchedulerSleepInfo *ssi;
    Process *current_process;
    Uint no;			/* Scheduler number for normal schedulers */
#ifdef ERTS_DIRTY_SCHEDULERS
    ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
#endif
# 658 "beam/erl_process.h"
    Port *current_port;
    ErtsRunQueue *run_queue;
    int virtual_reds;
    int cpu_id;			/* >= 0 when bound */
    ErtsAuxWorkData aux_work_data;
    ErtsAtomCacheMap atom_cache_map;

    ErtsMonotonicTime last_monotonic_time;
    int check_time_reds;

    Uint32 thr_id;
    Uint64 unique;
    Uint64 ref;

    ErtsSchedAllocData alloc_data;

    struct {
	Uint64 out;
	Uint64 in;
    } io;

    Uint64 reductions;
    ErtsSchedWallTime sched_wall_time;
    ErtsGCInfo gc_info;
    ErtsPortTaskHandle nosuspend_port_task_handle;

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
    erts_alloc_verify_func_t verify_unused_temp_alloc;
    Allctr_t *verify_unused_temp_alloc_data;
#endif
# 688 "beam/erl_process.h"
};

typedef union {
    ErtsSchedulerData esd;
    char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
} ErtsAlignedSchedulerData;

extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data;
#ifdef ERTS_DIRTY_SCHEDULERS
extern ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data;
extern ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data;
#endif
# 700 "beam/erl_process.h"

#ifndef ERTS_SMP
extern ErtsSchedulerData *erts_scheduler_data;
#endif
# 704 "beam/erl_process.h"

#ifdef ERTS_SCHED_FAIR
#define ERTS_SCHED_FAIR_YIELD() ETHR_YIELD()
#else
# 708 "beam/erl_process.h"
#define ERTS_SCHED_FAIR 0
#define ERTS_SCHED_FAIR_YIELD()
#endif
# 711 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_smp_lc_runq_is_locked(ErtsRunQueue *);
#endif
# 715 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP
void erts_empty_runq(ErtsRunQueue *rq);
void erts_non_empty_runq(ErtsRunQueue *rq);
#endif
# 722 "beam/erl_process.h"


/*
 * Run queue locked during modifications. We use atomic ops since
 * other threads peek at values without run queue lock.
 */

ERTS_GLB_INLINE void erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
ERTS_GLB_INLINE void erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_smp_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);

#ifdef ERTS_SMP
    if (len == 0)
	erts_non_empty_runq(rq);
#endif
# 748 "beam/erl_process.h"
    len++;
    if (rq->max_len < len)
	rq->max_len = len;
    ASSERT(len > 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))) == 0);
	erts_smp_atomic32_read_bor_nob(&rq->flags,
				       (erts_aint32_t) (1 << prio));
    }
    len++;
    if (rqi->max_len < len)
	rqi->max_len = len;

    erts_smp_atomic32_set_relb(&rqi->len, len);
}

ERTS_GLB_INLINE void
erts_smp_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rq->len);
    len--;
    ASSERT(len >= 0);
    erts_smp_atomic32_set_nob(&rq->len, len);

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    len--;
    ASSERT(len >= 0);
    if (len == 0) {
	ASSERT((erts_smp_atomic32_read_nob(&rq->flags)
		& ((erts_aint32_t) (1 << prio))));
	erts_smp_atomic32_read_band_nob(&rq->flags,
					~((erts_aint32_t) (1 << prio)));
    }
    erts_smp_atomic32_set_relb(&rqi->len, len);

}

ERTS_GLB_INLINE void
erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
{
    erts_aint32_t len;

    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));

    len = erts_smp_atomic32_read_dirty(&rqi->len);
    ASSERT(rqi->max_len >= len);
    rqi->max_len = len;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 807 "beam/erl_process.h"

#define RUNQ_READ_LEN(X) erts_smp_atomic32_read_nob((X))

#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
# 811 "beam/erl_process.h"

/*
 * Process Specific Data.
 *
 * NOTE: Only use PSD for very rarely used data.
 */

#define ERTS_PSD_ERROR_HANDLER			0
#define ERTS_PSD_SAVED_CALLS_BUF		1
#define ERTS_PSD_SCHED_ID			2
#define ERTS_PSD_CALL_TIME_BP			3
#define ERTS_PSD_DELAYED_GC_TASK_QS		4
#define ERTS_PSD_NIF_TRAP_EXPORT		5

#define ERTS_PSD_SIZE				6

typedef struct {
    void *data[ERTS_PSD_SIZE];
} ErtsPSD;

#ifdef ERTS_ENABLE_LOCK_CHECK
#define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)

#define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
#define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS

#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN

#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN

typedef struct {
    ErtsProcLocks get_locks;
    ErtsProcLocks set_locks;
} ErtsLcPSDLocks;

extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];

#endif
# 860 "beam/erl_process.h"

#define ERTS_SCHED_STAT_MODIFY_DISABLE		1
#define ERTS_SCHED_STAT_MODIFY_ENABLE		2
#define ERTS_SCHED_STAT_MODIFY_CLEAR		3

typedef struct {
    erts_smp_spinlock_t lock;
    int enabled;
    struct {
	Eterm name;
	Uint total_executed;
	Uint executed;
	Uint total_migrated;
	Uint migrated;
    } prio[ERTS_NO_PRIO_LEVELS];
} erts_sched_stat_t;

extern erts_sched_stat_t erts_sched_stat;

typedef struct {
    Eterm reason;
    ErlHeapFragment *bp;
} ErtsPendExit;

typedef struct ErtsProcSysTask_ ErtsProcSysTask;
typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;

#ifdef ERTS_SMP

typedef struct ErtsPendingSuspend_ ErtsPendingSuspend;
struct ErtsPendingSuspend_ {
    ErtsPendingSuspend *next;
    ErtsPendingSuspend *end;
    Eterm pid;
    void (*handle_func)(Process *suspendee,
			ErtsProcLocks suspendee_locks,
			int suspendee_alive,
			Eterm pid);
};

#endif
# 901 "beam/erl_process.h"


/* Defines to ease the change of memory architecture */
#  define HEAP_START(p)     (p)->heap
#  define HEAP_TOP(p)       (p)->htop
#  define HEAP_LIMIT(p)     (p)->stop
#  define HEAP_END(p)       (p)->hend
#  define HEAP_SIZE(p)      (p)->heap_sz
#  define STACK_START(p)    (p)->hend
#  define STACK_TOP(p)      (p)->stop
#  define STACK_END(p)      (p)->htop
#  define HIGH_WATER(p)     (p)->high_water
#  define OLD_HEND(p)       (p)->old_hend
#  define OLD_HTOP(p)       (p)->old_htop
#  define OLD_HEAP(p)       (p)->old_heap
#  define GEN_GCS(p)        (p)->gen_gcs
#  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
#  define FLAGS(p)          (p)->flags
#  define MBUF(p)           (p)->mbuf
#  define HALLOC_MBUF(p)    (p)->halloc_mbuf
#  define MBUF_SIZE(p)      (p)->mbuf_sz
#  define MSO(p)            (p)->off_heap
#  define MIN_HEAP_SIZE(p)  (p)->min_heap_size

#  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
#  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
#  define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature
#  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
#  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap

struct process {
    ErtsPTabElementCommon common; /* *Need* to be first in struct */

    /* All fields in the PCB that differs between different heap
     * architectures, have been moved to the end of this struct to
     * make sure that as few offsets as possible differ. Different
     * offsets between memory architectures in this struct, means that
     * native code have to use functions instead of constants.
     */

    Eterm* htop;		/* Heap top */
    Eterm* stop;		/* Stack top */
    Eterm* heap;		/* Heap start */
    Eterm* hend;		/* Heap end */
    Uint heap_sz;		/* Size of heap in words */
    Uint min_heap_size;         /* Minimum size of heap (in words). */
    Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */

#if !defined(NO_FPE_SIGNALS) || defined(HIPE)
    volatile unsigned long fp_exception;
#endif
# 952 "beam/erl_process.h"

#ifdef HIPE
    /* HiPE-specific process fields. Put it early in struct process,
       to enable smaller & faster addressing modes on the x86. */
    struct hipe_process_state hipe;
#endif
# 958 "beam/erl_process.h"

    /*
     * Saved x registers.
     */
    Uint arity;			/* Number of live argument registers (only valid
				 * when process is *not* running).
				 */
    Eterm* arg_reg;		/* Pointer to argument registers. */
    unsigned max_arg_reg;	/* Maximum number of argument registers available. */
    Eterm def_arg_reg[6];	/* Default array for argument registers. */

    BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
    BeamInstr* i;		/* Program counter for threaded code. */
    Sint catches;		/* Number of catches on stack */
    Sint fcalls;		/* 
				 * Number of reductions left to execute.
				 * Only valid for the current process.
				 */
    Uint32 rcount;		/* suspend count */
    int  schedule_count;	/* Times left to reschedule a low prio process */
    Uint reds;			/* No of reductions for this process  */
    Eterm group_leader;		/* Pid in charge
				   (can be boxed) */
    Uint flags;			/* Trap exit, etc (no trace flags anymore) */
    Eterm fvalue;		/* Exit & Throw value (failure reason) */
    Uint freason;		/* Reason for detected failure */
    Eterm ftrace;		/* Latest exception stack trace dump */

    Process *next;		/* Pointer to next process in run queue */

    struct ErtsNodesMonitor_ *nodes_monitors;

    ErtsSuspendMonitor *suspend_monitors; /* Processes suspended by
					     this process via
					     erlang:suspend_process/1 */

    ErlMessageQueue msg;	/* Message queue */

    ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
#ifdef ERTS_BTM_ACCESSOR_SUPPORT
    ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
#endif
# 1000 "beam/erl_process.h"

    ProcDict  *dictionary;       /* Process dictionary, may be NULL */

    Uint seq_trace_clock;
    Uint seq_trace_lastcnt;
    Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */

#ifdef USE_VM_PROBES
    Eterm dt_utag;              /* Place to store the dynamc trace user tag */
    Uint dt_utag_flags;         /* flag field for the dt_utag */
#endif
# 1011 "beam/erl_process.h"
    union {
	void *terminate;
	BeamInstr initial[3];	/* Initial module(0), function(1), arity(2), often used instead
				   of pointer to funcinfo instruction, hence the BeamInstr datatype */
    } u;
    BeamInstr* current;		/* Current Erlang function, part of the funcinfo:
				 * module(0), function(1), arity(2)
				 * (module and functions are tagged atoms;
				 * arity an untagged integer). BeamInstr * because it references code
				 */
    
    /*
     * Information mainly for post-mortem use (erl crash dump).
     */
    Eterm parent;		/* Pid of process that created this process. */
    erts_approx_time_t approx_started; /* Time when started. */

    Uint32 static_flags;        /* Flags that do *not* change */

    /* This is the place, where all fields that differs between memory
     * architectures, have gone to.
     */

    Eterm *high_water;
    Eterm *old_hend;            /* Heap pointers for generational GC. */
    Eterm *old_htop;
    Eterm *old_heap;
    Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
    Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
    ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
    ErlHeapFragment* mbuf;	/* Pointer to message buffer list */
    Uint mbuf_sz;		/* Size of all message buffers */
    ErtsPSD *psd;		/* Rarely used process specific data */

    Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
    Uint64 bin_vheap_mature;	/* Virtual heap block size for binaries */
    Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
    Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */

    ErtsProcSysTaskQs *sys_task_qs;

    erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */

#ifdef ERTS_SMP
    ErlMessageInQueue msg_inq;
    ErtsPendExit pending_exit;
    erts_proc_lock_t lock;
    ErtsSchedulerData *scheduler_data;
    Eterm suspendee;
    ErtsPendingSuspend *pending_suspenders;
    erts_smp_atomic_t run_queue;
#ifdef HIPE
    struct hipe_process_state_smp hipe_smp;
#endif
# 1065 "beam/erl_process.h"
#endif
# 1066 "beam/erl_process.h"

#ifdef CHECK_FOR_HOLES
    Eterm* last_htop;		/* No need to scan the heap below this point. */
    ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
#endif
# 1071 "beam/erl_process.h"

#ifdef DEBUG
    Eterm* last_old_htop;	/*
				 * No need to scan the old heap below this point
				 * when looking for invalid pointers into the new heap or
				 * heap fragments.
				 */
#endif
# 1079 "beam/erl_process.h"

#ifdef FORCE_HEAP_FRAGS
    Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
    Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
#endif
# 1084 "beam/erl_process.h"
};

extern const Process erts_invalid_process;

#ifdef CHECK_FOR_HOLES
# define INIT_HOLE_CHECK(p)			\
do {						\
  (p)->last_htop = 0;				\
  (p)->last_mbuf = 0;				\
} while (0)

# define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
void erts_check_for_holes(Process* p);
#else
# 1098 "beam/erl_process.h"
# define INIT_HOLE_CHECK(p)
# define ERTS_HOLE_CHECK(p)
#endif
# 1101 "beam/erl_process.h"

/*
 * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
 * due to message buffers allocated outside the heap.
 * The larger the factor, the easier the process gets GCed.
 * On a small memory system with lots of processes, this makes a significant 
 * difference, especially since the GCs help fragmentation quite a bit too.
 */
#if defined(SMALL_MEMORY)
#define MBUF_GC_FACTOR 4
#else
# 1112 "beam/erl_process.h"
#define MBUF_GC_FACTOR 1
#endif
# 1114 "beam/erl_process.h"

#define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)

#if ERTS_NO_PROC_PRIO_LEVELS > 4
#  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
#endif
# 1120 "beam/erl_process.h"

#define ERTS_PSFLGS_PRIO_BITS 2
#define ERTS_PSFLGS_PRIO_MASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)

#define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
#define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)

#define ERTS_PSFLGS_QMASK_BITS 4
#define ERTS_PSFLGS_QMASK \
    ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
#define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
    ERTS_PSFLGS_ZERO_BIT_OFFSET

#define ERTS_PSFLG_BIT(N) \
    (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))

/*
 * ACT_PRIO -> Active prio, i.e., currently active prio. This
 *             prio may be higher than user prio.
 * USR_PRIO -> User prio. i.e., prio the user has set.
 * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
 *             enqueued in. 
 */
#define ERTS_PSFLGS_ACT_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
#define ERTS_PSFLGS_USR_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
#define ERTS_PSFLGS_PRQ_PRIO_MASK \
    (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
#define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
#define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
#define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
#define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
#define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
#define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
#define ERTS_PSFLG_PENDING_EXIT		ERTS_PSFLG_BIT(6)
#define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
#define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
#define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
#define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
#define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
#define ERTS_PSFLG_BOUND		ERTS_PSFLG_BIT(12)
#define ERTS_PSFLG_TRAP_EXIT		ERTS_PSFLG_BIT(13)
#define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
#define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
#define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
#define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(18)
#define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(19)
#define ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q	ERTS_PSFLG_BIT(20)
#define ERTS_PSFLG_DIRTY_IO_PROC_IN_Q	ERTS_PSFLG_BIT(21)
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22)
#else
# 1177 "beam/erl_process.h"
#define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 18)
#endif
# 1179 "beam/erl_process.h"

#define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
					 | ERTS_PSFLG_IN_PRQ_HIGH	\
					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
					 | ERTS_PSFLG_IN_PRQ_LOW)

#define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
#define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
    (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)

/*
 * Static flags that do not change after process creation.
 */
#define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)

/* The sequential tracing token is a tuple of size 5:
 *
 *    {Flags, Label, Serial, Sender}
 */

#define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
#define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
#define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
#define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
#define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
#define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))

/* used when we have unit32 token */
#define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
#define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
#define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
#define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
#define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
#define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))

/*
 * Possible flags for the flags field in ErlSpawnOpts below.
 */

#define SPO_LINK 1
#define SPO_USE_ARGS 2
#define SPO_MONITOR 4
#define SPO_SYSTEM_PROC 8

/*
 * The following struct contains options for a process to be spawned.
 */
typedef struct {
    Uint flags;
    int error_code;		/* Error code returned from create_process(). */
    Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */

    /*
     * The following items are only initialized if the SPO_USE_ARGS flag is set.
     */
    Uint min_heap_size;		/* Minimum heap size (must be a valued returned
				 * from next_heap_size()).  */
    Uint min_vheap_size;	/* Minimum virtual heap size  */
    int priority;		/* Priority for process. */
    Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
    int scheduler;
} ErlSpawnOpts;

/*
 * The KILL_CATCHES(p) macro kills pending catches for process p.
 */

#define KILL_CATCHES(p) (p)->catches = -1

/* Shrink heap fragment from _last_ HAlloc.
*/
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
{
    ErlHeapFragment* hf = MBUF(p);

    ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size));

    hf->used_size = hp - hf->mem;
}	
#endif /* inline */
# 1264 "beam/erl_process.h"

Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
#ifdef CHECK_FOR_HOLES
Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz);
#endif
# 1269 "beam/erl_process.h"

extern Uint erts_default_process_flags;
extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx;
/* If any of the erts_system_monitor_* variables are set (enabled),
** erts_system_monitor must be != NIL, to allow testing on just
** the erts_system_monitor_* variables.
*/
extern Eterm erts_system_monitor;
extern Uint erts_system_monitor_long_gc;
extern Uint erts_system_monitor_long_schedule;
extern Uint erts_system_monitor_large_heap;
struct erts_system_monitor_flags_t {
	 unsigned int busy_port : 1;
    unsigned int busy_dist_port : 1;
};
extern struct erts_system_monitor_flags_t erts_system_monitor_flags;

/* system_profile, same rules as for system_monitor.
	erts_profile must be != NIL when 
	erts_profile_* is set. */

extern Eterm erts_system_profile;
struct erts_system_profile_flags_t {
    unsigned int scheduler : 1;
    unsigned int runnable_procs : 1;
    unsigned int runnable_ports : 1;
    unsigned int exclusive : 1;
};
extern struct erts_system_profile_flags_t erts_system_profile_flags;
extern int erts_system_profile_ts_type;

/* process flags */
#define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
#define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
#define F_TIMO               (1 <<  2) /* Set if timeout */
#define F_HEAP_GROW          (1 <<  3)
#define F_NEED_FULLSWEEP     (1 <<  4)
#define F_USING_DB           (1 <<  5) /* If have created tables */
#define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
#define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
#define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
#define F_P2PNR_RESCHED      (1 <<  9) /* Process has been rescheduled via erts_pid2proc_not_running() */
#define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
#define F_DISABLE_GC         (1 << 11) /* Disable GC */

#define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0

#define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* process trace_flags */

#define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
#define F_SENSITIVE          F_TRACE_FLAG(0)
#define F_TRACE_SEND         F_TRACE_FLAG(1)
#define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
#define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
#define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
#define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
#define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
#define F_TRACE_CALLS        F_TRACE_FLAG(7)
#define F_TRACE_PROCS        F_TRACE_FLAG(8)
#define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
#define F_TRACE_SCHED        F_TRACE_FLAG(10)
#define F_TRACE_GC           F_TRACE_FLAG(11)
#define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
#define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
#define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
#define F_TRACER             F_TRACE_FLAG(15) /* May be (has been) tracer */
#define F_EXCEPTION_TRACE    F_TRACE_FLAG(16) /* May have exception trace on stack */

/* port trace flags, currently the same as process trace flags */
#define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
#define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
#define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
#define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
#define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)

#define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
#ifdef DEBUG
#  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
#else
# 1355 "beam/erl_process.h"
#  define F_INITIAL_TRACE_FLAGS 0
#endif
# 1357 "beam/erl_process.h"

/* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
#define F_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)

#define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                     | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
		     | F_TRACE_SCHED_EXIT)

#define ERTS_TRACEE_MODIFIER_FLAGS \
  (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO)
#define ERTS_PORT_TRACEE_FLAGS \
  (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
#define ERTS_PROC_TRACEE_FLAGS \
  ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)

#define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))

/* Sequential trace flags */

/* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
#define SEQ_TRACE_TIMESTAMP_MASK \
    (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#define SEQ_TRACE_SEND     (1 << 0)
#define SEQ_TRACE_RECEIVE  (1 << 1)
#define SEQ_TRACE_PRINT    (1 << 2)

#define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3

#define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
#define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)

#ifdef USE_VM_PROBES
#define DT_UTAG_PERMANENT (1 << 0)
#define DT_UTAG_SPREADING (1 << 1)
#define DT_UTAG(P) ((P)->dt_utag)
#define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
#endif
# 1405 "beam/erl_process.h"

/* Option flags to erts_send_exit_signal() */
#define ERTS_XSIG_FLG_IGN_KILL		(((Uint32) 1) << 0)
#define ERTS_XSIG_FLG_NO_IGN_NORMAL	(((Uint32) 1) << 1)

#define CANCEL_TIMER(P)					\
    do {						\
	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
	    if ((P)->flags & F_INSLPQUEUE)		\
		erts_cancel_proc_timer((P));		\
	    else					\
		(P)->flags &= ~F_TIMO;			\
	}						\
    } while (0)

#if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
#define ERTS_NUM_DIRTY_RUNQS 2
#else
# 1423 "beam/erl_process.h"
#define ERTS_NUM_DIRTY_RUNQS 0
#endif
# 1425 "beam/erl_process.h"

#define ERTS_RUNQ_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues),			\
   &erts_aligned_run_queues[(IX)].runq)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
  (-(ERTS_NUM_DIRTY_RUNQS) <= (IX) && (IX) < 0)
#define ERTS_DIRTY_RUNQ_IX(IX)						\
  (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
   &erts_aligned_run_queues[(IX)].runq)
#define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[-1].runq)
#define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[-2].runq)
#define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ)->ix == -1)
#define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ)->ix == -2)
#else
# 1440 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#endif
# 1442 "beam/erl_process.h"
#define ERTS_SCHEDULER_IX(IX)						\
  (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
   &erts_aligned_scheduler_data[(IX)].esd)
#ifdef ERTS_DIRTY_SCHEDULERS
#define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
   &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
  (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
   &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
#define ERTS_DIRTY_SCHEDULER_NO(ESDP)					\
  ((ESDP)->dirty_no.s.num)
#define ERTS_DIRTY_SCHEDULER_TYPE(ESDP)					\
  ((ESDP)->dirty_no.s.type)
#ifdef ERTS_SMP
#define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
  ((ESDP)->dirty_no.s.num != 0)
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
    ((ESDP)->dirty_no.s.type == 0)
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
    ((ESDP)->dirty_no.s.type == 1)
#else
# 1464 "beam/erl_process.h"
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1468 "beam/erl_process.h"
#else
# 1469 "beam/erl_process.h"
#define ERTS_RUNQ_IX_IS_DIRTY(IX) 0
#define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0
#define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) 0
#endif
# 1474 "beam/erl_process.h"

void erts_pre_init_process(void);
void erts_late_init_process(void);
void erts_early_init_scheduling(int);
void erts_init_scheduling(int, int
#ifdef ERTS_DIRTY_SCHEDULERS
			  , int, int, int
#endif
# 1482 "beam/erl_process.h"
			  );

int erts_set_gc_state(Process *c_p, int enable);
Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
Eterm erts_system_check_request(Process *c_p);
Eterm erts_gc_info_request(Process *c_p);
Uint64 erts_get_proc_interval(void);
Uint64 erts_ensure_later_proc_interval(Uint64);
Uint64 erts_step_proc_interval(void);

int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
void erts_destroy_nif_export(void *); /* see erl_nif.c */

ErtsProcList *erts_proclist_create(Process *);
void erts_proclist_destroy(ErtsProcList *);

ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int
erts_proclist_same(ErtsProcList *plp, Process *p)
{
    return (plp->pid == p->common.id
	    && (plp->started_interval
		== p->common.u.alive.started_interval));
}

ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
					       ErtsProcList *element)
{
    if (!*list)
	element->next = element->prev = element;
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
    *list = element;
}

ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
					      ErtsProcList *element)
{
    if (!*list) {
	element->next = element->prev = element;
	*list = element;
    }
    else {
	element->prev = (*list)->prev;
	element->next = *list;
	element->prev->next = element;
	element->next->prev = element;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
{
    return list;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
{
    if (!list)
	return NULL;
    else
	return list->prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *next;
    ASSERT(list && element);
    next = element->next;
    return list == next ? NULL : next;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
						      ErtsProcList *element)
{
    ErtsProcList *prev;
    ASSERT(list && element);
    prev = element->prev;
    return list == element ? NULL : prev;
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = *list;
	if (res == *list)
	    *list = NULL;
	else
	    *list = res->next;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
{
    if (!*list)
	return NULL;
    else {
	ErtsProcList *res = (*list)->prev;
	if (res == *list)
	    *list = NULL;
	res->next->prev = res->prev;
	res->prev->next = res->next;
	return res;
    }
}

ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
					ErtsProcList **list_last)
{
    if (!*list_first) {
	if (list_last)
	    *list_last = NULL;
	return 0;
    }
    else {
	if (list_last)
	    *list_last = (*list_first)->prev;
	(*list_first)->prev->next = NULL;
	(*list_first)->prev = NULL;
	return !0;
    }
}

ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
					  ErtsProcList *element)
{
    ASSERT(list && *list);
    if (*list == element) {
	*list = element->next;
	if (*list == element)
	    *list = NULL;
    }
    element->next->prev = element->prev;
    element->prev->next = element->next;
}

ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
{
    return list == NULL;
}

ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
					   ErtsProcList *element)
{
    ASSERT(list && element);
    return list == element;
}

ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
					  ErtsProcList *element)
{
    ASSERT(list && element);
    return list->prev == element;
}

#endif
# 1663 "beam/erl_process.h"

int erts_sched_set_wakeup_other_thresold(char *str);
int erts_sched_set_wakeup_other_type(char *str);
int erts_sched_set_busy_wait_threshold(char *str);
int erts_sched_set_wake_cleanup_threshold(char *);

void erts_schedule_thr_prgr_later_op(void (*)(void *),
				     void *,
				     ErtsThrPrgrLaterOp *);
void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
					     void *,
					     ErtsThrPrgrLaterOp *,
					     UWord);

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
int erts_dbg_check_halloc_lock(Process *p);
#endif
# 1680 "beam/erl_process.h"
#ifdef DEBUG
void erts_dbg_multi_scheduling_return_trap(Process *, Eterm);
#endif
# 1683 "beam/erl_process.h"
int erts_get_max_no_executing_schedulers(void);
#if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS)
ErtsSchedSuspendResult
erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int);
#endif
# 1688 "beam/erl_process.h"
#ifdef ERTS_SMP
ErtsSchedSuspendResult
erts_set_schedulers_online(Process *p,
			   ErtsProcLocks plocks,
			   Sint new_no,
			   Sint *old_no
#ifdef ERTS_DIRTY_SCHEDULERS
			   , int dirty_only
#endif
# 1697 "beam/erl_process.h"
			   );
ErtsSchedSuspendResult
erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int);
int erts_is_multi_scheduling_blocked(void);
Eterm erts_multi_scheduling_blockers(Process *);
void erts_start_schedulers(void);
void erts_alloc_notify_delayed_dealloc(int);
void erts_alloc_ensure_handle_delayed_dealloc_call(int);
#ifdef ERTS_SMP
void erts_notify_canceled_timer(ErtsSchedulerData *, int);
#endif
# 1708 "beam/erl_process.h"
void erts_smp_notify_check_children_needed(void);
#endif
# 1710 "beam/erl_process.h"
#if ERTS_USE_ASYNC_READY_Q
void erts_notify_check_async_ready_queue(void *);
#endif
# 1713 "beam/erl_process.h"
#ifdef ERTS_SMP
void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
void erts_notify_finish_breakpointing(Process* p);
#endif
# 1717 "beam/erl_process.h"
void erts_schedule_misc_aux_work(int sched_id,
				 void (*func)(void *),
				 void *arg);
void erts_schedule_multi_misc_aux_work(int ignore_self,
				       int max_sched,
				       void (*func)(void *),
				       void *arg);
erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
void erts_sched_notify_check_cpu_bind(void);
Uint erts_active_schedulers(void);
void erts_init_process(int, int, int);
Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm);
Uint erts_run_queues_len(Uint *, int, int);
void erts_add_to_runq(Process *);
Eterm erts_bound_schedulers_term(Process *c_p);
Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
Eterm erts_get_schedulers_binds(Process *c_p);
Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
Eterm erts_bind_schedulers(Process *c_p, Eterm how);
ErtsRunQueue *erts_schedid2runq(Uint);
Process *schedule(Process*, int);
void erts_schedule_misc_op(void (*)(void *), void *);
Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
void erts_do_exit_process(Process*, Eterm);
void erts_continue_exit_process(Process *);
/* Begin System profile */
Uint erts_runnable_process_count(void);
/* End System profile */
void erts_init_empty_process(Process *p);
void erts_cleanup_empty_process(Process* p);
#ifdef DEBUG
void erts_debug_verify_clean_empty_process(Process* p);
#endif
# 1750 "beam/erl_process.h"
void erts_stack_dump(int to, void *to_arg, Process *);
void erts_limited_stack_trace(int to, void *to_arg, Process *);
void erts_program_counter_info(int to, void *to_arg, Process *);
void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp);
void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg);
void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg);

Eterm erts_get_process_priority(Process *p);
Eterm erts_set_process_priority(Process *p, Eterm prio);

Uint erts_get_total_context_switches(void);
void erts_get_total_reductions(Uint *, Uint *);
void erts_get_exact_total_reductions(Process *, Uint *, Uint *);

Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);

void erts_sched_stat_modify(int what);
Eterm erts_sched_stat_term(Process *p, int total);

void erts_free_proc(Process *);

void erts_suspend(Process*, ErtsProcLocks, Port*);
void erts_resume(Process*, ErtsProcLocks);
int erts_resume_processes(ErtsProcList *);

int erts_send_exit_signal(Process *,
			  Eterm,
			  Process *,
			  ErtsProcLocks *,
			  Eterm,
			  Eterm,
			  Process *,
			  Uint32);
#ifdef ERTS_SMP
void erts_handle_pending_exit(Process *, ErtsProcLocks);
#define ERTS_PROC_PENDING_EXIT(P) \
    (ERTS_PSFLG_PENDING_EXIT & erts_smp_atomic32_read_acqb(&(P)->state))
#else
# 1788 "beam/erl_process.h"
#define ERTS_PROC_PENDING_EXIT(P) 0
#endif
# 1790 "beam/erl_process.h"

void erts_deep_process_dump(int, void *);

Eterm erts_get_reader_groups_map(Process *c_p);
Eterm erts_debug_reader_groups_map(Process *c_p, int groups);

Uint erts_debug_nbalance(void);

#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)

int erts_debug_wait_completed(Process *c_p, int flags);

Uint erts_process_memory(Process *c_p);

#ifdef ERTS_SMP
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data)
#else
# 1809 "beam/erl_process.h"
#  define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data)
#  define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data)
#endif
# 1812 "beam/erl_process.h"

#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
do {										\
    ErtsSchedulerData *esdp__ = ((P)						\
				 ? ERTS_PROC_GET_SCHDATA((Process *) (P))	\
				 : erts_get_scheduler_data());			\
    if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
	esdp__->verify_unused_temp_alloc(					\
	    esdp__->verify_unused_temp_alloc_data);				\
} while (0)
#else
# 1824 "beam/erl_process.h"
#  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
#endif
# 1826 "beam/erl_process.h"

#if defined(ERTS_SMP) || defined(USE_THREADS)
ErtsSchedulerData *erts_get_scheduler_data(void);
#else
# 1830 "beam/erl_process.h"
ERTS_GLB_INLINE ErtsSchedulerData *erts_get_scheduler_data(void);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
ErtsSchedulerData *erts_get_scheduler_data(void)
{
    return erts_scheduler_data;
}
#endif
# 1839 "beam/erl_process.h"
#endif
# 1840 "beam/erl_process.h"

void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);

ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
{
    /* No barrier needed, due to msg lock */
    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
    if (!(state & ERTS_PSFLG_ACTIVE))
	erts_schedule_process(p, state, locks);
}
#endif
# 1854 "beam/erl_process.h"

#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)

#define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 1858 "beam/erl_process.h"
# 1859 "beam/erl_process.h"
#undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__

#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)				\
do {									\
    if ((L))								\
	ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked((RQ)));		\
    else								\
	ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked((RQ)));		\
} while (0)
#else
# 1869 "beam/erl_process.h"
#define ERTS_SMP_LC_CHK_RUNQ_LOCK(RQ, L)
#endif
# 1871 "beam/erl_process.h"

void *erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data);

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix);
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *new);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void *
erts_psd_get(Process *p, int ix)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].get_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1894 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    return p->psd ? p->psd->data[ix] : NULL;
}


/*
 * NOTE: erts_psd_set() might release and reacquire locks on 'p'.
 */
ERTS_GLB_INLINE void *
erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data)
{
#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
    ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
    if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
	ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking());
    else {
	locks &= erts_psd_required_locks[ix].set_locks;
	ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
			   || erts_thr_progress_is_blocking());
    }
#endif
# 1915 "beam/erl_process.h"
    ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
    if (p->psd) {
	void *old = p->psd->data[ix];
	p->psd->data[ix] = data;
	return old;
    }
    else {
	if (!data)
	    return NULL;
	else
	    return erts_psd_set_init(p, plocks, ix, data);
    }
}

#endif
# 1930 "beam/erl_process.h"

#define ERTS_PROC_SCHED_ID(P, L, ID) \
  ((UWord) erts_psd_set((P), (L), ERTS_PSD_SCHED_ID, (void *) (ID)))

#define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
  ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
#define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \
  ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))

#define ERTS_PROC_GET_CALL_TIME(P) \
  ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \
  ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))

#define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
    ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
#define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, L, PBT) \
    ((ErtsProcSysTaskQs *) erts_psd_set((P), (L), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))

#define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
    erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
#define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, L, NTE) \
    erts_psd_set((P), (L), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))


ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p,
						  ErtsProcLocks plocks,
						  Eterm handler);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Eterm
erts_proc_get_error_handler(Process *p)
{
    void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
    if (!val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) val)));
	return (Eterm) (UWord) val;
    }
}

ERTS_GLB_INLINE Eterm
erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, Eterm handler)
{
    void *old_val;
    void *new_val;
    ASSERT(is_atom(handler));
    new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
    old_val = erts_psd_set(p, plocks, ERTS_PSD_ERROR_HANDLER, new_val);
    if (!old_val)
	return am_error_handler;
    else {
	ASSERT(is_atom(((Eterm) (UWord) old_val)));
	return (Eterm) (UWord) old_val;
    }
}

#endif
# 1990 "beam/erl_process.h"

#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS

#ifdef ERTS_SMP

#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 1995 "beam/erl_process.h"
# 1996 "beam/erl_process.h"

extern erts_atomic_t erts_migration_paths;

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
int erts_get_sched_util(ErtsRunQueue *rq,
			int initially_locked,
			int short_interval);
#endif
# 2004 "beam/erl_process.h"


ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
							 int prio);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths_managed(void)
{
    return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
}

ERTS_GLB_INLINE ErtsMigrationPaths *
erts_get_migration_paths(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_get_migration_paths_managed();
    else
	return NULL;
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
{
    ErtsMigrationPaths *mps = erts_get_migration_paths();
    ErtsMigrationPath *mp;
    Uint32 flags;

    if (!mps)
	return NULL;

    mp = &mps->mpath[c_rq->ix];
    flags = mp->flags;

    if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
	int len;

	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
	    /* force emigration */
	    return mp->prio[prio].runq;
	}

	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
	    /*
	     * Run queue was inactive at last balance. Verify that
	     * it still is before forcing emigration.
	     */
	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
		return mp->prio[prio].runq;
	}

#if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
	if (mp->sched_util) {
	    ErtsRunQueue *rq = mp->prio[prio].runq;
	    /* No migration if other is non-empty */
	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
		return rq;
	    }
	}
	else
#endif
# 2069 "beam/erl_process.h"
	{

	    if (prio == ERTS_PORT_PRIO_LEVEL)
		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
	    else
		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);

	    if (len > mp->prio[prio].limit.this) {
		ErtsRunQueue *n_rq = mp->prio[prio].runq;
		if (n_rq) {
		    if (prio == ERTS_PORT_PRIO_LEVEL)
			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
		    else
			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);

		    if (len < mp->prio[prio].limit.other)
			return n_rq;
		}
	    }
	}
    }
    return NULL;
}

#endif
# 2094 "beam/erl_process.h"

#endif
# 2096 "beam/erl_process.h"

#endif
# 2098 "beam/erl_process.h"

ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE Process *erts_get_current_process(void);
ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p);
ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
ERTS_GLB_INLINE void erts_smp_runq_lock(ErtsRunQueue *rq);
ERTS_GLB_INLINE int erts_smp_runq_trylock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_runq_unlock(ErtsRunQueue *rq);
ERTS_GLB_INLINE void erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
ERTS_GLB_INLINE void erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
ERTS_GLB_INLINE void erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE
int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
{
    if (!esdp)
	esdp = erts_get_scheduler_data();
    ASSERT(esdp);
    return esdp->cpu_id >= 0;
}

ERTS_GLB_INLINE
Process *erts_get_current_process(void)
{
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
    return esdp ? esdp->current_process : NULL;
}

ERTS_GLB_INLINE
Eterm erts_get_current_pid(void)
{
    Process *proc = erts_get_current_process();
    return proc ? proc->common.id : THE_NON_VALUE;
}

ERTS_GLB_INLINE
Uint erts_get_scheduler_id(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp = erts_get_scheduler_data();
#ifdef ERTS_DIRTY_SCHEDULERS
    if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
	return 0;
    else
#endif
# 2148 "beam/erl_process.h"
	return esdp ? esdp->no : (Uint) 0;
#else
# 2150 "beam/erl_process.h"
    return erts_get_scheduler_data() ? (Uint) 1 : (Uint) 0;
#endif
# 2152 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_proc(Process *p)
{
#ifdef ERTS_SMP
    ASSERT(ERTS_AINT_NULL != erts_atomic_read_nob(&p->run_queue));
    return (ErtsRunQueue *) erts_atomic_read_nob(&p->run_queue);
#else
# 2161 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2163 "beam/erl_process.h"
}

ERTS_GLB_INLINE ErtsRunQueue *
erts_get_runq_current(ErtsSchedulerData *esdp)
{
    ASSERT(!esdp || esdp == erts_get_scheduler_data());
#ifdef ERTS_SMP
    if (!esdp)
	esdp = erts_get_scheduler_data();
    return esdp->run_queue;
#else
# 2174 "beam/erl_process.h"
    return ERTS_RUNQ_IX(0);
#endif
# 2176 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_lock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_lock(&rq->mtx);
#endif
# 2184 "beam/erl_process.h"
}

ERTS_GLB_INLINE int
erts_smp_runq_trylock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    return erts_smp_mtx_trylock(&rq->mtx);
#else
# 2192 "beam/erl_process.h"
    return 0;
#endif
# 2194 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runq_unlock(ErtsRunQueue *rq)
{
#ifdef ERTS_SMP
    erts_smp_mtx_unlock(&rq->mtx);
#endif
# 2202 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&rq->mtx));
    if (xrq != rq) {
	if (erts_smp_mtx_trylock(&xrq->mtx) == EBUSY) {
	    if (rq < xrq)
		erts_smp_mtx_lock(&xrq->mtx);
	    else {
		erts_smp_mtx_unlock(&rq->mtx);
		erts_smp_mtx_lock(&xrq->mtx);
		erts_smp_mtx_lock(&rq->mtx);
	    }
	}
    }
#endif
# 2221 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
{
#ifdef ERTS_SMP
    if (xrq != rq)
	erts_smp_mtx_unlock(&xrq->mtx);
#endif
# 2230 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    if (rq1 == rq2)
	erts_smp_mtx_lock(&rq1->mtx);
    else if (rq1 < rq2) {
	erts_smp_mtx_lock(&rq1->mtx);
	erts_smp_mtx_lock(&rq2->mtx);	
    }
    else {
	erts_smp_mtx_lock(&rq2->mtx);
	erts_smp_mtx_lock(&rq1->mtx);	
    }
#endif
# 2248 "beam/erl_process.h"
}

ERTS_GLB_INLINE void
erts_smp_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
{
#ifdef ERTS_SMP
    ASSERT(rq1 && rq2);
    erts_smp_mtx_unlock(&rq1->mtx);
    if (rq1 != rq2)
	erts_smp_mtx_unlock(&rq2->mtx);
#endif
# 2259 "beam/erl_process.h"
}

#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2262 "beam/erl_process.h"

ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE ErtsAtomCacheMap *
erts_get_atom_cache_map(Process *c_p)
{
    ErtsSchedulerData *esdp = (c_p
			       ? ERTS_PROC_GET_SCHDATA(c_p)
			       : erts_get_scheduler_data());
    ASSERT(esdp);
    return &esdp->atom_cache_map;
}
#endif
# 2276 "beam/erl_process.h"

Process *erts_pid2proc_suspend(Process *,
			       ErtsProcLocks,
			       Eterm,
			       ErtsProcLocks);
#ifdef ERTS_SMP

Process *erts_pid2proc_not_running(Process *,
				   ErtsProcLocks,
				   Eterm,
				   ErtsProcLocks);
Process *erts_pid2proc_nropt(Process *c_p,
			     ErtsProcLocks c_p_locks,
			     Eterm pid,
			     ErtsProcLocks pid_locks);
extern int erts_disable_proc_not_running_opt;

#ifdef DEBUG
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P) \
    do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
#else
# 2297 "beam/erl_process.h"
#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)
#endif
# 2299 "beam/erl_process.h"

#else /* !ERTS_SMP */
# 2301 "beam/erl_process.h"

#define ERTS_SMP_ASSERT_IS_NOT_EXITING(P)

#define erts_pid2proc_not_running erts_pid2proc
#define erts_pid2proc_nropt erts_pid2proc

#endif
# 2308 "beam/erl_process.h"

#define ERTS_PROC_IS_EXITING(P) \
    (ERTS_PSFLG_EXITING & erts_smp_atomic32_read_acqb(&(P)->state))


/* Minimum NUMBER of processes for a small system to start */
#define ERTS_MIN_PROCESSES		1024
#if defined(ERTS_SMP) && ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
#undef ERTS_MIN_PROCESSES
#define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
#endif
# 2319 "beam/erl_process.h"

void erts_smp_notify_inc_runq(ErtsRunQueue *runq);

void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);

#ifdef ERTS_SMP
void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
{
    erts_aint32_t flags;
    ERTS_THR_MEMORY_BARRIER;
    flags = erts_smp_atomic32_read_nob(&ssi->flags);
    if (flags & ERTS_SSI_FLG_SLEEPING) {
	flags = erts_smp_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
	erts_sched_finish_poke(ssi, flags);
    }
}


#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 2344 "beam/erl_process.h"

#endif /* #ifdef ERTS_SMP */
# 2346 "beam/erl_process.h"

#if 0 /* expanded by -frewrite-includes */
#include "erl_process_lock.h"
#endif /* expanded by -frewrite-includes */
# 2347 "beam/erl_process.h"
# 2348 "beam/erl_process.h"

#undef ERTS_INCLUDE_SCHEDULER_INTERNALS

#endif
# 2352 "beam/erl_process.h"


void erts_halt(int code);
extern erts_smp_atomic32_t erts_halt_progress;
extern int erts_halt_code;
# 25 "beam/dist.h" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_node_tables.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/dist.h"
# 26 "beam/dist.h"
#if 0 /* expanded by -frewrite-includes */
#include "zlib.h"
#endif /* expanded by -frewrite-includes */
# 26 "beam/dist.h"
# 1 "/usr/include/zlib.h" 1 3 4
/* zlib.h -- interface of the 'zlib' general purpose compression library
  version 1.2.8, April 28th, 2013

  Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler

  This software is provided 'as-is', without any express or implied
  warranty.  In no event will the authors be held liable for any damages
  arising from the use of this software.

  Permission is granted to anyone to use this software for any purpose,
  including commercial applications, and to alter it and redistribute it
  freely, subject to the following restrictions:

  1. The origin of this software must not be misrepresented; you must not
     claim that you wrote the original software. If you use this software
     in a product, an acknowledgment in the product documentation would be
     appreciated but is not required.
  2. Altered source versions must be plainly marked as such, and must not be
     misrepresented as being the original software.
  3. This notice may not be removed or altered from any source distribution.

  Jean-loup Gailly        Mark Adler
  jloup@gzip.org          madler@alumni.caltech.edu


  The data format used by the zlib library is described by RFCs (Request for
  Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950
  (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format).
*/

#ifndef ZLIB_H
#define ZLIB_H

#if 0 /* expanded by -frewrite-includes */
#include "zconf.h"
#endif /* expanded by -frewrite-includes */
# 34 "/usr/include/zlib.h" 3 4
# 1 "/usr/include/zconf.h" 1 3 4
/* zconf.h -- configuration of the zlib compression library
 * Copyright (C) 1995-2013 Jean-loup Gailly.
 * For conditions of distribution and use, see copyright notice in zlib.h
 */

/* @(#) $FreeBSD: head/lib/libz/zconf.h 250261 2013-05-05 06:20:49Z delphij $ */

#ifndef ZCONF_H
#define ZCONF_H

/*
 * If you *really* need a unique prefix for all types and library functions,
 * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it.
 * Even better than compiling with -DZ_PREFIX would be to use configure to set
 * this permanently in zconf.h using "./configure --zprefix".
 */
#ifdef Z_PREFIX     /* may be set to #if 1 by ./configure */
#  define Z_PREFIX_SET

/* all linked symbols */
#  define _dist_code            z__dist_code
#  define _length_code          z__length_code
#  define _tr_align             z__tr_align
#  define _tr_flush_bits        z__tr_flush_bits
#  define _tr_flush_block       z__tr_flush_block
#  define _tr_init              z__tr_init
#  define _tr_stored_block      z__tr_stored_block
#  define _tr_tally             z__tr_tally
#  define adler32               z_adler32
#  define adler32_combine       z_adler32_combine
#  define adler32_combine64     z_adler32_combine64
#  ifndef Z_SOLO
#    define compress              z_compress
#    define compress2             z_compress2
#    define compressBound         z_compressBound
#  endif
# 37 "/usr/include/zconf.h" 3 4
#  define crc32                 z_crc32
#  define crc32_combine         z_crc32_combine
#  define crc32_combine64       z_crc32_combine64
#  define deflate               z_deflate
#  define deflateBound          z_deflateBound
#  define deflateCopy           z_deflateCopy
#  define deflateEnd            z_deflateEnd
#  define deflateInit2_         z_deflateInit2_
#  define deflateInit_          z_deflateInit_
#  define deflateParams         z_deflateParams
#  define deflatePending        z_deflatePending
#  define deflatePrime          z_deflatePrime
#  define deflateReset          z_deflateReset
#  define deflateResetKeep      z_deflateResetKeep
#  define deflateSetDictionary  z_deflateSetDictionary
#  define deflateSetHeader      z_deflateSetHeader
#  define deflateTune           z_deflateTune
#  define deflate_copyright     z_deflate_copyright
#  define get_crc_table         z_get_crc_table
#  ifndef Z_SOLO
#    define gz_error              z_gz_error
#    define gz_intmax             z_gz_intmax
#    define gz_strwinerror        z_gz_strwinerror
#    define gzbuffer              z_gzbuffer
#    define gzclearerr            z_gzclearerr
#    define gzclose               z_gzclose
#    define gzclose_r             z_gzclose_r
#    define gzclose_w             z_gzclose_w
#    define gzdirect              z_gzdirect
#    define gzdopen               z_gzdopen
#    define gzeof                 z_gzeof
#    define gzerror               z_gzerror
#    define gzflush               z_gzflush
#    define gzgetc                z_gzgetc
#    define gzgetc_               z_gzgetc_
#    define gzgets                z_gzgets
#    define gzoffset              z_gzoffset
#    define gzoffset64            z_gzoffset64
#    define gzopen                z_gzopen
#    define gzopen64              z_gzopen64
#    ifdef _WIN32
#      define gzopen_w              z_gzopen_w
#    endif
# 80 "/usr/include/zconf.h" 3 4
#    define gzprintf              z_gzprintf
#    define gzvprintf             z_gzvprintf
#    define gzputc                z_gzputc
#    define gzputs                z_gzputs
#    define gzread                z_gzread
#    define gzrewind              z_gzrewind
#    define gzseek                z_gzseek
#    define gzseek64              z_gzseek64
#    define gzsetparams           z_gzsetparams
#    define gztell                z_gztell
#    define gztell64              z_gztell64
#    define gzungetc              z_gzungetc
#    define gzwrite               z_gzwrite
#  endif
# 94 "/usr/include/zconf.h" 3 4
#  define inflate               z_inflate
#  define inflateBack           z_inflateBack
#  define inflateBackEnd        z_inflateBackEnd
#  define inflateBackInit_      z_inflateBackInit_
#  define inflateCopy           z_inflateCopy
#  define inflateEnd            z_inflateEnd
#  define inflateGetHeader      z_inflateGetHeader
#  define inflateInit2_         z_inflateInit2_
#  define inflateInit_          z_inflateInit_
#  define inflateMark           z_inflateMark
#  define inflatePrime          z_inflatePrime
#  define inflateReset          z_inflateReset
#  define inflateReset2         z_inflateReset2
#  define inflateSetDictionary  z_inflateSetDictionary
#  define inflateGetDictionary  z_inflateGetDictionary
#  define inflateSync           z_inflateSync
#  define inflateSyncPoint      z_inflateSyncPoint
#  define inflateUndermine      z_inflateUndermine
#  define inflateResetKeep      z_inflateResetKeep
#  define inflate_copyright     z_inflate_copyright
#  define inflate_fast          z_inflate_fast
#  define inflate_table         z_inflate_table
#  ifndef Z_SOLO
#    define uncompress            z_uncompress
#  endif
# 119 "/usr/include/zconf.h" 3 4
#  define zError                z_zError
#  ifndef Z_SOLO
#    define zcalloc               z_zcalloc
#    define zcfree                z_zcfree
#  endif
# 124 "/usr/include/zconf.h" 3 4
#  define zlibCompileFlags      z_zlibCompileFlags
#  define zlibVersion           z_zlibVersion

/* all zlib typedefs in zlib.h and zconf.h */
#  define Byte                  z_Byte
#  define Bytef                 z_Bytef
#  define alloc_func            z_alloc_func
#  define charf                 z_charf
#  define free_func             z_free_func
#  ifndef Z_SOLO
#    define gzFile                z_gzFile
#  endif
# 136 "/usr/include/zconf.h" 3 4
#  define gz_header             z_gz_header
#  define gz_headerp            z_gz_headerp
#  define in_func               z_in_func
#  define intf                  z_intf
#  define out_func              z_out_func
#  define uInt                  z_uInt
#  define uIntf                 z_uIntf
#  define uLong                 z_uLong
#  define uLongf                z_uLongf
#  define voidp                 z_voidp
#  define voidpc                z_voidpc
#  define voidpf                z_voidpf

/* all zlib structs in zlib.h and zconf.h */
#  define gz_header_s           z_gz_header_s
#  define internal_state        z_internal_state

#endif
# 154 "/usr/include/zconf.h" 3 4

#if defined(__MSDOS__) && !defined(MSDOS)
#  define MSDOS
#endif
# 158 "/usr/include/zconf.h" 3 4
#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2)
#  define OS2
#endif
# 161 "/usr/include/zconf.h" 3 4
#if defined(_WINDOWS) && !defined(WINDOWS)
#  define WINDOWS
#endif
# 164 "/usr/include/zconf.h" 3 4
#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__)
#  ifndef WIN32
#    define WIN32
#  endif
# 168 "/usr/include/zconf.h" 3 4
#endif
# 169 "/usr/include/zconf.h" 3 4
#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32)
#  if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__)
#    ifndef SYS16BIT
#      define SYS16BIT
#    endif
# 174 "/usr/include/zconf.h" 3 4
#  endif
# 175 "/usr/include/zconf.h" 3 4
#endif
# 176 "/usr/include/zconf.h" 3 4

/*
 * Compile with -DMAXSEG_64K if the alloc function cannot allocate more
 * than 64k bytes at a time (needed on systems with 16-bit int).
 */
#ifdef SYS16BIT
#  define MAXSEG_64K
#endif
# 184 "/usr/include/zconf.h" 3 4
#ifdef MSDOS
#  define UNALIGNED_OK
#endif
# 187 "/usr/include/zconf.h" 3 4

#ifdef __STDC_VERSION__
#  ifndef STDC
#    define STDC
#  endif
# 192 "/usr/include/zconf.h" 3 4
#  if __STDC_VERSION__ >= 199901L
#    ifndef STDC99
#      define STDC99
#    endif
# 196 "/usr/include/zconf.h" 3 4
#  endif
# 197 "/usr/include/zconf.h" 3 4
#endif
# 198 "/usr/include/zconf.h" 3 4
#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus))
#  define STDC
#endif
# 201 "/usr/include/zconf.h" 3 4
#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__))
#  define STDC
#endif
# 204 "/usr/include/zconf.h" 3 4
#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32))
#  define STDC
#endif
# 207 "/usr/include/zconf.h" 3 4
#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__))
#  define STDC
#endif
# 210 "/usr/include/zconf.h" 3 4

#if defined(__OS400__) && !defined(STDC)    /* iSeries (formerly AS/400). */
#  define STDC
#endif
# 214 "/usr/include/zconf.h" 3 4

#ifndef STDC
#  ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */
#    define const       /* note: need a more gentle solution here */
#  endif
# 219 "/usr/include/zconf.h" 3 4
#endif
# 220 "/usr/include/zconf.h" 3 4

#if defined(ZLIB_CONST) && !defined(z_const)
#  define z_const const
#else
# 224 "/usr/include/zconf.h" 3 4
#  define z_const
#endif
# 226 "/usr/include/zconf.h" 3 4

/* Some Mac compilers merge all .h files incorrectly: */
#if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__)
#  define NO_DUMMY_DECL
#endif
# 231 "/usr/include/zconf.h" 3 4

/* Maximum value for memLevel in deflateInit2 */
#ifndef MAX_MEM_LEVEL
#  ifdef MAXSEG_64K
#    define MAX_MEM_LEVEL 8
#  else
# 237 "/usr/include/zconf.h" 3 4
#    define MAX_MEM_LEVEL 9
#  endif
# 239 "/usr/include/zconf.h" 3 4
#endif
# 240 "/usr/include/zconf.h" 3 4

/* Maximum value for windowBits in deflateInit2 and inflateInit2.
 * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files
 * created by gzip. (Files created by minigzip can still be extracted by
 * gzip.)
 */
#ifndef MAX_WBITS
#  define MAX_WBITS   15 /* 32K LZ77 window */
#endif
# 249 "/usr/include/zconf.h" 3 4

/* The memory requirements for deflate are (in bytes):
            (1 << (windowBits+2)) +  (1 << (memLevel+9))
 that is: 128K for windowBits=15  +  128K for memLevel = 8  (default values)
 plus a few kilobytes for small objects. For example, if you want to reduce
 the default memory requirements from 256K to 128K, compile with
     make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7"
 Of course this will generally degrade compression (there's no free lunch).

   The memory requirements for inflate are (in bytes) 1 << windowBits
 that is, 32K for windowBits=15 (default value) plus a few kilobytes
 for small objects.
*/

                        /* Type declarations */

#ifndef OF /* function prototypes */
#  ifdef STDC
#    define OF(args)  args
#  else
# 269 "/usr/include/zconf.h" 3 4
#    define OF(args)  ()
#  endif
# 271 "/usr/include/zconf.h" 3 4
#endif
# 272 "/usr/include/zconf.h" 3 4

#ifndef Z_ARG /* function prototypes for stdarg */
#  if defined(STDC) || defined(Z_HAVE_STDARG_H)
#    define Z_ARG(args)  args
#  else
# 277 "/usr/include/zconf.h" 3 4
#    define Z_ARG(args)  ()
#  endif
# 279 "/usr/include/zconf.h" 3 4
#endif
# 280 "/usr/include/zconf.h" 3 4

/* The following definitions for FAR are needed only for MSDOS mixed
 * model programming (small or medium model with some far allocations).
 * This was tested only with MSC; for other MSDOS compilers you may have
 * to define NO_MEMCPY in zutil.h.  If you don't need the mixed model,
 * just define FAR to be empty.
 */
#ifdef SYS16BIT
#  if defined(M_I86SM) || defined(M_I86MM)
     /* MSC small or medium model */
#    define SMALL_MEDIUM
#    ifdef _MSC_VER
#      define FAR _far
#    else
# 294 "/usr/include/zconf.h" 3 4
#      define FAR far
#    endif
# 296 "/usr/include/zconf.h" 3 4
#  endif
# 297 "/usr/include/zconf.h" 3 4
#  if (defined(__SMALL__) || defined(__MEDIUM__))
     /* Turbo C small or medium model */
#    define SMALL_MEDIUM
#    ifdef __BORLANDC__
#      define FAR _far
#    else
# 303 "/usr/include/zconf.h" 3 4
#      define FAR far
#    endif
# 305 "/usr/include/zconf.h" 3 4
#  endif
# 306 "/usr/include/zconf.h" 3 4
#endif
# 307 "/usr/include/zconf.h" 3 4

#if defined(WINDOWS) || defined(WIN32)
   /* If building or using zlib as a DLL, define ZLIB_DLL.
    * This is not mandatory, but it offers a little performance increase.
    */
#  ifdef ZLIB_DLL
#    if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500))
#      ifdef ZLIB_INTERNAL
#        define ZEXTERN extern __declspec(dllexport)
#      else
# 317 "/usr/include/zconf.h" 3 4
#        define ZEXTERN extern __declspec(dllimport)
#      endif
# 319 "/usr/include/zconf.h" 3 4
#    endif
# 320 "/usr/include/zconf.h" 3 4
#  endif  /* ZLIB_DLL */
# 321 "/usr/include/zconf.h" 3 4
   /* If building or using zlib with the WINAPI/WINAPIV calling convention,
    * define ZLIB_WINAPI.
    * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI.
    */
#  ifdef ZLIB_WINAPI
#    ifdef FAR
#      undef FAR
#    endif
# 329 "/usr/include/zconf.h" 3 4
#if 0 /* expanded by -frewrite-includes */
#    include <windows.h>
#endif /* expanded by -frewrite-includes */
# 329 "/usr/include/zconf.h" 3 4
# 330 "/usr/include/zconf.h" 3 4
     /* No need for _export, use ZLIB.DEF instead. */
     /* For complete Windows compatibility, use WINAPI, not __stdcall. */
#    define ZEXPORT WINAPI
#    ifdef WIN32
#      define ZEXPORTVA WINAPIV
#    else
# 336 "/usr/include/zconf.h" 3 4
#      define ZEXPORTVA FAR CDECL
#    endif
# 338 "/usr/include/zconf.h" 3 4
#  endif
# 339 "/usr/include/zconf.h" 3 4
#endif
# 340 "/usr/include/zconf.h" 3 4

#if defined (__BEOS__)
#  ifdef ZLIB_DLL
#    ifdef ZLIB_INTERNAL
#      define ZEXPORT   __declspec(dllexport)
#      define ZEXPORTVA __declspec(dllexport)
#    else
# 347 "/usr/include/zconf.h" 3 4
#      define ZEXPORT   __declspec(dllimport)
#      define ZEXPORTVA __declspec(dllimport)
#    endif
# 350 "/usr/include/zconf.h" 3 4
#  endif
# 351 "/usr/include/zconf.h" 3 4
#endif
# 352 "/usr/include/zconf.h" 3 4

#ifndef ZEXTERN
#  define ZEXTERN extern
#endif
# 356 "/usr/include/zconf.h" 3 4
#ifndef ZEXPORT
#  define ZEXPORT
#endif
# 359 "/usr/include/zconf.h" 3 4
#ifndef ZEXPORTVA
#  define ZEXPORTVA
#endif
# 362 "/usr/include/zconf.h" 3 4

#ifndef FAR
#  define FAR
#endif
# 366 "/usr/include/zconf.h" 3 4

#if !defined(__MACTYPES__)
typedef unsigned char  Byte;  /* 8 bits */
#endif
# 370 "/usr/include/zconf.h" 3 4
typedef unsigned int   uInt;  /* 16 bits or more */
typedef unsigned long  uLong; /* 32 bits or more */

#ifdef SMALL_MEDIUM
   /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */
#  define Bytef Byte FAR
#else
# 377 "/usr/include/zconf.h" 3 4
   typedef Byte  FAR Bytef;
#endif
# 379 "/usr/include/zconf.h" 3 4
typedef char  FAR charf;
typedef int   FAR intf;
typedef uInt  FAR uIntf;
typedef uLong FAR uLongf;

#ifdef STDC
   typedef void const *voidpc;
   typedef void FAR   *voidpf;
   typedef void       *voidp;
#else
# 389 "/usr/include/zconf.h" 3 4
   typedef Byte const *voidpc;
   typedef Byte FAR   *voidpf;
   typedef Byte       *voidp;
#endif
# 393 "/usr/include/zconf.h" 3 4

#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC)
#if 0 /* expanded by -frewrite-includes */
#  include <limits.h>
#endif /* expanded by -frewrite-includes */
# 395 "/usr/include/zconf.h" 3 4
# 396 "/usr/include/zconf.h" 3 4
#  if (UINT_MAX == 0xffffffffUL)
#    define Z_U4 unsigned
#  elif (ULONG_MAX == 0xffffffffUL)
# 399 "/usr/include/zconf.h" 3 4
#    define Z_U4 unsigned long
#  elif (USHRT_MAX == 0xffffffffUL)
# 401 "/usr/include/zconf.h" 3 4
#    define Z_U4 unsigned short
#  endif
# 403 "/usr/include/zconf.h" 3 4
#endif
# 404 "/usr/include/zconf.h" 3 4

#ifdef Z_U4
   typedef Z_U4 z_crc_t;
#else
# 408 "/usr/include/zconf.h" 3 4
   typedef unsigned long z_crc_t;
#endif
# 410 "/usr/include/zconf.h" 3 4

#if 0    /* was set to #if 0 by ./configure */
#  define Z_HAVE_UNISTD_H
#endif
# 414 "/usr/include/zconf.h" 3 4

#ifdef HAVE_STDARG_H    /* may be set to #if 1 by ./configure */
#  define Z_HAVE_STDARG_H
#endif
# 418 "/usr/include/zconf.h" 3 4

#ifdef STDC
#  ifndef Z_SOLO
#if 0 /* expanded by -frewrite-includes */
#    include <sys/types.h>      /* for off_t */
#endif /* expanded by -frewrite-includes */
# 421 "/usr/include/zconf.h" 3 4
# 422 "/usr/include/zconf.h" 3 4
#  endif
# 423 "/usr/include/zconf.h" 3 4
#endif
# 424 "/usr/include/zconf.h" 3 4

#if defined(STDC) || defined(Z_HAVE_STDARG_H)
#  ifndef Z_SOLO
#if 0 /* expanded by -frewrite-includes */
#    include <stdarg.h>         /* for va_list */
#endif /* expanded by -frewrite-includes */
# 427 "/usr/include/zconf.h" 3 4
# 428 "/usr/include/zconf.h" 3 4
#  endif
# 429 "/usr/include/zconf.h" 3 4
#endif
# 430 "/usr/include/zconf.h" 3 4

#ifdef _WIN32
#  ifndef Z_SOLO
#if 0 /* expanded by -frewrite-includes */
#    include <stddef.h>         /* for wchar_t */
#endif /* expanded by -frewrite-includes */
# 433 "/usr/include/zconf.h" 3 4
# 434 "/usr/include/zconf.h" 3 4
#  endif
# 435 "/usr/include/zconf.h" 3 4
#endif
# 436 "/usr/include/zconf.h" 3 4

/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and
 * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even
 * though the former does not conform to the LFS document), but considering
 * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as
 * equivalently requesting no 64-bit operations
 */
#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1
#  undef _LARGEFILE64_SOURCE
#endif
# 446 "/usr/include/zconf.h" 3 4

#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H)
#  define Z_HAVE_UNISTD_H
#endif
# 450 "/usr/include/zconf.h" 3 4
#ifndef Z_SOLO
#  if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE)
#if 0 /* expanded by -frewrite-includes */
#    include <unistd.h>         /* for SEEK_*, off_t, and _LFS64_LARGEFILE */
#endif /* expanded by -frewrite-includes */
# 452 "/usr/include/zconf.h" 3 4
# 453 "/usr/include/zconf.h" 3 4
#    ifdef VMS
#if 0 /* expanded by -frewrite-includes */
#      include <unixio.h>       /* for off_t */
#endif /* expanded by -frewrite-includes */
# 454 "/usr/include/zconf.h" 3 4
# 455 "/usr/include/zconf.h" 3 4
#    endif
# 456 "/usr/include/zconf.h" 3 4
#    ifndef z_off_t
#      define z_off_t off_t
#    endif
# 459 "/usr/include/zconf.h" 3 4
#  endif
# 460 "/usr/include/zconf.h" 3 4
#endif
# 461 "/usr/include/zconf.h" 3 4

#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0
#  define Z_LFS64
#endif
# 465 "/usr/include/zconf.h" 3 4

#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64)
#  define Z_LARGE64
#endif
# 469 "/usr/include/zconf.h" 3 4

#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64)
#  define Z_WANT64
#endif
# 473 "/usr/include/zconf.h" 3 4

#if !defined(SEEK_SET) && !defined(Z_SOLO)
#  define SEEK_SET        0       /* Seek from beginning of file.  */
#  define SEEK_CUR        1       /* Seek from current position.  */
#  define SEEK_END        2       /* Set file pointer to EOF plus "offset" */
#endif
# 479 "/usr/include/zconf.h" 3 4

/*
 * This is hard-configured for FreeBSD.
 */
#define	z_off_t	off_t
#ifndef _FILE_OFFSET_BITS
#define _FILE_OFFSET_BITS 64
#endif
# 487 "/usr/include/zconf.h" 3 4

#ifndef z_off_t
#  define z_off_t long
#endif
# 491 "/usr/include/zconf.h" 3 4

#if !defined(_WIN32) && defined(Z_LARGE64)
#  define z_off64_t off64_t
#else
# 495 "/usr/include/zconf.h" 3 4
#  if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO)
#    define z_off64_t __int64
#  else
# 498 "/usr/include/zconf.h" 3 4
#    define z_off64_t z_off_t
#  endif
# 500 "/usr/include/zconf.h" 3 4
#endif
# 501 "/usr/include/zconf.h" 3 4

/* MVS linker does not support external names larger than 8 bytes */
#if defined(__MVS__)
  #pragma map(deflateInit_,"DEIN")
  #pragma map(deflateInit2_,"DEIN2")
  #pragma map(deflateEnd,"DEEND")
  #pragma map(deflateBound,"DEBND")
  #pragma map(inflateInit_,"ININ")
  #pragma map(inflateInit2_,"ININ2")
  #pragma map(inflateEnd,"INEND")
  #pragma map(inflateSync,"INSY")
  #pragma map(inflateSetDictionary,"INSEDI")
  #pragma map(compressBound,"CMBND")
  #pragma map(inflate_table,"INTABL")
  #pragma map(inflate_fast,"INFA")
  #pragma map(inflate_copyright,"INCOPY")
#endif
# 518 "/usr/include/zconf.h" 3 4

#endif /* ZCONF_H */
# 520 "/usr/include/zconf.h" 3 4
# 35 "/usr/include/zlib.h" 2 3 4

#ifdef __cplusplus
extern "C" {
#endif
# 39 "/usr/include/zlib.h" 3 4

#define ZLIB_VERSION "1.2.8"
#define ZLIB_VERNUM 0x1280
#define ZLIB_VER_MAJOR 1
#define ZLIB_VER_MINOR 2
#define ZLIB_VER_REVISION 8
#define ZLIB_VER_SUBREVISION 0

/*
    The 'zlib' compression library provides in-memory compression and
  decompression functions, including integrity checks of the uncompressed data.
  This version of the library supports only one compression method (deflation)
  but other algorithms will be added later and will have the same stream
  interface.

    Compression can be done in a single step if the buffers are large enough,
  or can be done by repeated calls of the compression function.  In the latter
  case, the application must provide more input and/or consume the output
  (providing more output space) before each call.

    The compressed data format used by default by the in-memory functions is
  the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped
  around a deflate stream, which is itself documented in RFC 1951.

    The library also supports reading and writing files in gzip (.gz) format
  with an interface similar to that of stdio using the functions that start
  with "gz".  The gzip format is different from the zlib format.  gzip is a
  gzip wrapper, documented in RFC 1952, wrapped around a deflate stream.

    This library can optionally read and write gzip streams in memory as well.

    The zlib format was designed to be compact and fast for use in memory
  and on communications channels.  The gzip format was designed for single-
  file compression on file systems, has a larger header than zlib to maintain
  directory information, and uses a different, slower check method than zlib.

    The library does not install any signal handler.  The decoder checks
  the consistency of the compressed data, so the library should never crash
  even in case of corrupted input.
*/

typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
typedef void   (*free_func)  OF((voidpf opaque, voidpf address));

struct internal_state;

typedef struct z_stream_s {
    z_const Bytef *next_in;     /* next input byte */
    uInt     avail_in;  /* number of bytes available at next_in */
    uLong    total_in;  /* total number of input bytes read so far */

    Bytef    *next_out; /* next output byte should be put there */
    uInt     avail_out; /* remaining free space at next_out */
    uLong    total_out; /* total number of bytes output so far */

    z_const char *msg;  /* last error message, NULL if no error */
    struct internal_state FAR *state; /* not visible by applications */

    alloc_func zalloc;  /* used to allocate the internal state */
    free_func  zfree;   /* used to free the internal state */
    voidpf     opaque;  /* private data object passed to zalloc and zfree */

    int     data_type;  /* best guess about the data type: binary or text */
    uLong   adler;      /* adler32 value of the uncompressed data */
    uLong   reserved;   /* reserved for future use */
} z_stream;

typedef z_stream FAR *z_streamp;

/*
     gzip header information passed to and from zlib routines.  See RFC 1952
  for more details on the meanings of these fields.
*/
typedef struct gz_header_s {
    int     text;       /* true if compressed data believed to be text */
    uLong   time;       /* modification time */
    int     xflags;     /* extra flags (not used when writing a gzip file) */
    int     os;         /* operating system */
    Bytef   *extra;     /* pointer to extra field or Z_NULL if none */
    uInt    extra_len;  /* extra field length (valid if extra != Z_NULL) */
    uInt    extra_max;  /* space at extra (only when reading header) */
    Bytef   *name;      /* pointer to zero-terminated file name or Z_NULL */
    uInt    name_max;   /* space at name (only when reading header) */
    Bytef   *comment;   /* pointer to zero-terminated comment or Z_NULL */
    uInt    comm_max;   /* space at comment (only when reading header) */
    int     hcrc;       /* true if there was or will be a header crc */
    int     done;       /* true when done reading gzip header (not used
                           when writing a gzip file) */
} gz_header;

typedef gz_header FAR *gz_headerp;

/*
     The application must update next_in and avail_in when avail_in has dropped
   to zero.  It must update next_out and avail_out when avail_out has dropped
   to zero.  The application must initialize zalloc, zfree and opaque before
   calling the init function.  All other fields are set by the compression
   library and must not be updated by the application.

     The opaque value provided by the application will be passed as the first
   parameter for calls of zalloc and zfree.  This can be useful for custom
   memory management.  The compression library attaches no meaning to the
   opaque value.

     zalloc must return Z_NULL if there is not enough memory for the object.
   If zlib is used in a multi-threaded application, zalloc and zfree must be
   thread safe.

     On 16-bit systems, the functions zalloc and zfree must be able to allocate
   exactly 65536 bytes, but will not be required to allocate more than this if
   the symbol MAXSEG_64K is defined (see zconf.h).  WARNING: On MSDOS, pointers
   returned by zalloc for objects of exactly 65536 bytes *must* have their
   offset normalized to zero.  The default allocation function provided by this
   library ensures this (see zutil.c).  To reduce memory requirements and avoid
   any allocation of 64K objects, at the expense of compression ratio, compile
   the library with -DMAX_WBITS=14 (see zconf.h).

     The fields total_in and total_out can be used for statistics or progress
   reports.  After compression, total_in holds the total size of the
   uncompressed data and may be saved for use in the decompressor (particularly
   if the decompressor wants to decompress everything in a single step).
*/

                        /* constants */

#define Z_NO_FLUSH      0
#define Z_PARTIAL_FLUSH 1
#define Z_SYNC_FLUSH    2
#define Z_FULL_FLUSH    3
#define Z_FINISH        4
#define Z_BLOCK         5
#define Z_TREES         6
/* Allowed flush values; see deflate() and inflate() below for details */

#define Z_OK            0
#define Z_STREAM_END    1
#define Z_NEED_DICT     2
#define Z_ERRNO        (-1)
#define Z_STREAM_ERROR (-2)
#define Z_DATA_ERROR   (-3)
#define Z_MEM_ERROR    (-4)
#define Z_BUF_ERROR    (-5)
#define Z_VERSION_ERROR (-6)
/* Return codes for the compression/decompression functions. Negative values
 * are errors, positive values are used for special but normal events.
 */

#define Z_NO_COMPRESSION         0
#define Z_BEST_SPEED             1
#define Z_BEST_COMPRESSION       9
#define Z_DEFAULT_COMPRESSION  (-1)
/* compression levels */

#define Z_FILTERED            1
#define Z_HUFFMAN_ONLY        2
#define Z_RLE                 3
#define Z_FIXED               4
#define Z_DEFAULT_STRATEGY    0
/* compression strategy; see deflateInit2() below for details */

#define Z_BINARY   0
#define Z_TEXT     1
#define Z_ASCII    Z_TEXT   /* for compatibility with 1.2.2 and earlier */
#define Z_UNKNOWN  2
/* Possible values of the data_type field (though see inflate()) */

#define Z_DEFLATED   8
/* The deflate compression method (the only one supported in this version) */

#define Z_NULL  0  /* for initializing zalloc, zfree, opaque */

#define zlib_version zlibVersion()
/* for compatibility with versions < 1.0.2 */


                        /* basic functions */

ZEXTERN const char * ZEXPORT zlibVersion OF((void));
/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
   If the first character differs, the library code actually used is not
   compatible with the zlib.h header file used by the application.  This check
   is automatically made by deflateInit and inflateInit.
 */

/*
ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));

     Initializes the internal stream state for compression.  The fields
   zalloc, zfree and opaque must be initialized before by the caller.  If
   zalloc and zfree are set to Z_NULL, deflateInit updates them to use default
   allocation functions.

     The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
   1 gives best speed, 9 gives best compression, 0 gives no compression at all
   (the input data is simply copied a block at a time).  Z_DEFAULT_COMPRESSION
   requests a default compromise between speed and compression (currently
   equivalent to level 6).

     deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
   memory, Z_STREAM_ERROR if level is not a valid compression level, or
   Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
   with the version assumed by the caller (ZLIB_VERSION).  msg is set to null
   if there is no error message.  deflateInit does not perform any compression:
   this will be done by deflate().
*/


ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
/*
    deflate compresses as much data as possible, and stops when the input
  buffer becomes empty or the output buffer becomes full.  It may introduce
  some output latency (reading input without producing any output) except when
  forced to flush.

    The detailed semantics are as follows.  deflate performs one or both of the
  following actions:

  - Compress more input starting at next_in and update next_in and avail_in
    accordingly.  If not all input can be processed (because there is not
    enough room in the output buffer), next_in and avail_in are updated and
    processing will resume at this point for the next call of deflate().

  - Provide more output starting at next_out and update next_out and avail_out
    accordingly.  This action is forced if the parameter flush is non zero.
    Forcing flush frequently degrades the compression ratio, so this parameter
    should be set only when necessary (in interactive applications).  Some
    output may be provided even if flush is not set.

    Before the call of deflate(), the application should ensure that at least
  one of the actions is possible, by providing more input and/or consuming more
  output, and updating avail_in or avail_out accordingly; avail_out should
  never be zero before the call.  The application can consume the compressed
  output when it wants, for example when the output buffer is full (avail_out
  == 0), or after each call of deflate().  If deflate returns Z_OK and with
  zero avail_out, it must be called again after making room in the output
  buffer because there might be more output pending.

    Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to
  decide how much data to accumulate before producing output, in order to
  maximize compression.

    If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
  flushed to the output buffer and the output is aligned on a byte boundary, so
  that the decompressor can get all input data available so far.  (In
  particular avail_in is zero after the call if enough output space has been
  provided before the call.) Flushing may degrade compression for some
  compression algorithms and so it should be used only when necessary.  This
  completes the current deflate block and follows it with an empty stored block
  that is three bits plus filler bits to the next byte, followed by four bytes
  (00 00 ff ff).

    If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the
  output buffer, but the output is not aligned to a byte boundary.  All of the
  input data so far will be available to the decompressor, as for Z_SYNC_FLUSH.
  This completes the current deflate block and follows it with an empty fixed
  codes block that is 10 bits long.  This assures that enough bytes are output
  in order for the decompressor to finish the block before the empty fixed code
  block.

    If flush is set to Z_BLOCK, a deflate block is completed and emitted, as
  for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to
  seven bits of the current block are held to be written as the next byte after
  the next deflate block is completed.  In this case, the decompressor may not
  be provided enough bits at this point in order to complete decompression of
  the data provided so far to the compressor.  It may need to wait for the next
  block to be emitted.  This is for advanced applications that need to control
  the emission of deflate blocks.

    If flush is set to Z_FULL_FLUSH, all output is flushed as with
  Z_SYNC_FLUSH, and the compression state is reset so that decompression can
  restart from this point if previous compressed data has been damaged or if
  random access is desired.  Using Z_FULL_FLUSH too often can seriously degrade
  compression.

    If deflate returns with avail_out == 0, this function must be called again
  with the same value of the flush parameter and more output space (updated
  avail_out), until the flush is complete (deflate returns with non-zero
  avail_out).  In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that
  avail_out is greater than six to avoid repeated flush markers due to
  avail_out == 0 on return.

    If the parameter flush is set to Z_FINISH, pending input is processed,
  pending output is flushed and deflate returns with Z_STREAM_END if there was
  enough output space; if deflate returns with Z_OK, this function must be
  called again with Z_FINISH and more output space (updated avail_out) but no
  more input data, until it returns with Z_STREAM_END or an error.  After
  deflate has returned Z_STREAM_END, the only possible operations on the stream
  are deflateReset or deflateEnd.

    Z_FINISH can be used immediately after deflateInit if all the compression
  is to be done in a single step.  In this case, avail_out must be at least the
  value returned by deflateBound (see below).  Then deflate is guaranteed to
  return Z_STREAM_END.  If not enough output space is provided, deflate will
  not return Z_STREAM_END, and it must be called again as described above.

    deflate() sets strm->adler to the adler32 checksum of all input read
  so far (that is, total_in bytes).

    deflate() may update strm->data_type if it can make a good guess about
  the input data type (Z_BINARY or Z_TEXT).  In doubt, the data is considered
  binary.  This field is only for information purposes and does not affect the
  compression algorithm in any manner.

    deflate() returns Z_OK if some progress has been made (more input
  processed or more output produced), Z_STREAM_END if all input has been
  consumed and all output has been produced (only when flush is set to
  Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
  if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible
  (for example avail_in or avail_out was zero).  Note that Z_BUF_ERROR is not
  fatal, and deflate() can be called again with more input and more output
  space to continue compressing.
*/


ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
/*
     All dynamically allocated data structures for this stream are freed.
   This function discards any unprocessed input and does not flush any pending
   output.

     deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
   stream state was inconsistent, Z_DATA_ERROR if the stream was freed
   prematurely (some input or output was discarded).  In the error case, msg
   may be set but then points to a static string (which must not be
   deallocated).
*/


/*
ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));

     Initializes the internal stream state for decompression.  The fields
   next_in, avail_in, zalloc, zfree and opaque must be initialized before by
   the caller.  If next_in is not Z_NULL and avail_in is large enough (the
   exact value depends on the compression method), inflateInit determines the
   compression method from the zlib header and allocates all data structures
   accordingly; otherwise the allocation will be deferred to the first call of
   inflate.  If zalloc and zfree are set to Z_NULL, inflateInit updates them to
   use default allocation functions.

     inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
   version assumed by the caller, or Z_STREAM_ERROR if the parameters are
   invalid, such as a null pointer to the structure.  msg is set to null if
   there is no error message.  inflateInit does not perform any decompression
   apart from possibly reading the zlib header if present: actual decompression
   will be done by inflate().  (So next_in and avail_in may be modified, but
   next_out and avail_out are unused and unchanged.) The current implementation
   of inflateInit() does not process any header information -- that is deferred
   until inflate() is called.
*/


ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
/*
    inflate decompresses as much data as possible, and stops when the input
  buffer becomes empty or the output buffer becomes full.  It may introduce
  some output latency (reading input without producing any output) except when
  forced to flush.

  The detailed semantics are as follows.  inflate performs one or both of the
  following actions:

  - Decompress more input starting at next_in and update next_in and avail_in
    accordingly.  If not all input can be processed (because there is not
    enough room in the output buffer), next_in is updated and processing will
    resume at this point for the next call of inflate().

  - Provide more output starting at next_out and update next_out and avail_out
    accordingly.  inflate() provides as much output as possible, until there is
    no more input data or no more space in the output buffer (see below about
    the flush parameter).

    Before the call of inflate(), the application should ensure that at least
  one of the actions is possible, by providing more input and/or consuming more
  output, and updating the next_* and avail_* values accordingly.  The
  application can consume the uncompressed output when it wants, for example
  when the output buffer is full (avail_out == 0), or after each call of
  inflate().  If inflate returns Z_OK and with zero avail_out, it must be
  called again after making room in the output buffer because there might be
  more output pending.

    The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH,
  Z_BLOCK, or Z_TREES.  Z_SYNC_FLUSH requests that inflate() flush as much
  output as possible to the output buffer.  Z_BLOCK requests that inflate()
  stop if and when it gets to the next deflate block boundary.  When decoding
  the zlib or gzip format, this will cause inflate() to return immediately
  after the header and before the first block.  When doing a raw inflate,
  inflate() will go ahead and process the first block, and will return when it
  gets to the end of that block, or when it runs out of data.

    The Z_BLOCK option assists in appending to or combining deflate streams.
  Also to assist in this, on return inflate() will set strm->data_type to the
  number of unused bits in the last byte taken from strm->next_in, plus 64 if
  inflate() is currently decoding the last block in the deflate stream, plus
  128 if inflate() returned immediately after decoding an end-of-block code or
  decoding the complete header up to just before the first byte of the deflate
  stream.  The end-of-block will not be indicated until all of the uncompressed
  data from that block has been written to strm->next_out.  The number of
  unused bits may in general be greater than seven, except when bit 7 of
  data_type is set, in which case the number of unused bits will be less than
  eight.  data_type is set as noted here every time inflate() returns for all
  flush options, and so can be used to determine the amount of currently
  consumed input in bits.

    The Z_TREES option behaves as Z_BLOCK does, but it also returns when the
  end of each deflate block header is reached, before any actual data in that
  block is decoded.  This allows the caller to determine the length of the
  deflate block header for later use in random access within a deflate block.
  256 is added to the value of strm->data_type when inflate() returns
  immediately after reaching the end of the deflate block header.

    inflate() should normally be called until it returns Z_STREAM_END or an
  error.  However if all decompression is to be performed in a single step (a
  single call of inflate), the parameter flush should be set to Z_FINISH.  In
  this case all pending input is processed and all pending output is flushed;
  avail_out must be large enough to hold all of the uncompressed data for the
  operation to complete.  (The size of the uncompressed data may have been
  saved by the compressor for this purpose.) The use of Z_FINISH is not
  required to perform an inflation in one step.  However it may be used to
  inform inflate that a faster approach can be used for the single inflate()
  call.  Z_FINISH also informs inflate to not maintain a sliding window if the
  stream completes, which reduces inflate's memory footprint.  If the stream
  does not complete, either because not all of the stream is provided or not
  enough output space is provided, then a sliding window will be allocated and
  inflate() can be called again to continue the operation as if Z_NO_FLUSH had
  been used.

     In this implementation, inflate() always flushes as much output as
  possible to the output buffer, and always uses the faster approach on the
  first call.  So the effects of the flush parameter in this implementation are
  on the return value of inflate() as noted below, when inflate() returns early
  when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of
  memory for a sliding window when Z_FINISH is used.

     If a preset dictionary is needed after this call (see inflateSetDictionary
  below), inflate sets strm->adler to the Adler-32 checksum of the dictionary
  chosen by the compressor and returns Z_NEED_DICT; otherwise it sets
  strm->adler to the Adler-32 checksum of all output produced so far (that is,
  total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described
  below.  At the end of the stream, inflate() checks that its computed adler32
  checksum is equal to that saved by the compressor and returns Z_STREAM_END
  only if the checksum is correct.

    inflate() can decompress and check either zlib-wrapped or gzip-wrapped
  deflate data.  The header type is detected automatically, if requested when
  initializing with inflateInit2().  Any information contained in the gzip
  header is not retained, so applications that need that information should
  instead use raw inflate, see inflateInit2() below, or inflateBack() and
  perform their own processing of the gzip header and trailer.  When processing
  gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output
  producted so far.  The CRC-32 is checked against the gzip trailer.

    inflate() returns Z_OK if some progress has been made (more input processed
  or more output produced), Z_STREAM_END if the end of the compressed data has
  been reached and all uncompressed output has been produced, Z_NEED_DICT if a
  preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
  corrupted (input stream not conforming to the zlib format or incorrect check
  value), Z_STREAM_ERROR if the stream structure was inconsistent (for example
  next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory,
  Z_BUF_ERROR if no progress is possible or if there was not enough room in the
  output buffer when Z_FINISH is used.  Note that Z_BUF_ERROR is not fatal, and
  inflate() can be called again with more input and more output space to
  continue decompressing.  If Z_DATA_ERROR is returned, the application may
  then call inflateSync() to look for a good compression block if a partial
  recovery of the data is desired.
*/


ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
/*
     All dynamically allocated data structures for this stream are freed.
   This function discards any unprocessed input and does not flush any pending
   output.

     inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
   was inconsistent.  In the error case, msg may be set but then points to a
   static string (which must not be deallocated).
*/


                        /* Advanced functions */

/*
    The following functions are needed only in some special applications.
*/

/*
ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
                                     int  level,
                                     int  method,
                                     int  windowBits,
                                     int  memLevel,
                                     int  strategy));

     This is another version of deflateInit with more compression options.  The
   fields next_in, zalloc, zfree and opaque must be initialized before by the
   caller.

     The method parameter is the compression method.  It must be Z_DEFLATED in
   this version of the library.

     The windowBits parameter is the base two logarithm of the window size
   (the size of the history buffer).  It should be in the range 8..15 for this
   version of the library.  Larger values of this parameter result in better
   compression at the expense of memory usage.  The default value is 15 if
   deflateInit is used instead.

     windowBits can also be -8..-15 for raw deflate.  In this case, -windowBits
   determines the window size.  deflate() will then generate raw deflate data
   with no zlib header or trailer, and will not compute an adler32 check value.

     windowBits can also be greater than 15 for optional gzip encoding.  Add
   16 to windowBits to write a simple gzip header and trailer around the
   compressed data instead of a zlib wrapper.  The gzip header will have no
   file name, no extra data, no comment, no modification time (set to zero), no
   header crc, and the operating system will be set to 255 (unknown).  If a
   gzip stream is being written, strm->adler is a crc32 instead of an adler32.

     The memLevel parameter specifies how much memory should be allocated
   for the internal compression state.  memLevel=1 uses minimum memory but is
   slow and reduces compression ratio; memLevel=9 uses maximum memory for
   optimal speed.  The default value is 8.  See zconf.h for total memory usage
   as a function of windowBits and memLevel.

     The strategy parameter is used to tune the compression algorithm.  Use the
   value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
   filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no
   string match), or Z_RLE to limit match distances to one (run-length
   encoding).  Filtered data consists mostly of small values with a somewhat
   random distribution.  In this case, the compression algorithm is tuned to
   compress them better.  The effect of Z_FILTERED is to force more Huffman
   coding and less string matching; it is somewhat intermediate between
   Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY.  Z_RLE is designed to be almost as
   fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data.  The
   strategy parameter only affects the compression ratio but not the
   correctness of the compressed output even if it is not set appropriately.
   Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler
   decoder for special applications.

     deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
   memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid
   method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is
   incompatible with the version assumed by the caller (ZLIB_VERSION).  msg is
   set to null if there is no error message.  deflateInit2 does not perform any
   compression: this will be done by deflate().
*/

ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
                                             const Bytef *dictionary,
                                             uInt  dictLength));
/*
     Initializes the compression dictionary from the given byte sequence
   without producing any compressed output.  When using the zlib format, this
   function must be called immediately after deflateInit, deflateInit2 or
   deflateReset, and before any call of deflate.  When doing raw deflate, this
   function must be called either before any call of deflate, or immediately
   after the completion of a deflate block, i.e. after all input has been
   consumed and all output has been delivered when using any of the flush
   options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH.  The
   compressor and decompressor must use exactly the same dictionary (see
   inflateSetDictionary).

     The dictionary should consist of strings (byte sequences) that are likely
   to be encountered later in the data to be compressed, with the most commonly
   used strings preferably put towards the end of the dictionary.  Using a
   dictionary is most useful when the data to be compressed is short and can be
   predicted with good accuracy; the data can then be compressed better than
   with the default empty dictionary.

     Depending on the size of the compression data structures selected by
   deflateInit or deflateInit2, a part of the dictionary may in effect be
   discarded, for example if the dictionary is larger than the window size
   provided in deflateInit or deflateInit2.  Thus the strings most likely to be
   useful should be put at the end of the dictionary, not at the front.  In
   addition, the current implementation of deflate will use at most the window
   size minus 262 bytes of the provided dictionary.

     Upon return of this function, strm->adler is set to the adler32 value
   of the dictionary; the decompressor may later use this value to determine
   which dictionary has been used by the compressor.  (The adler32 value
   applies to the whole dictionary even if only a subset of the dictionary is
   actually used by the compressor.) If a raw deflate was requested, then the
   adler32 value is not computed and strm->adler is not set.

     deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
   parameter is invalid (e.g.  dictionary being Z_NULL) or the stream state is
   inconsistent (for example if deflate has already been called for this stream
   or if not at a block boundary for raw deflate).  deflateSetDictionary does
   not perform any compression: this will be done by deflate().
*/

ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
                                    z_streamp source));
/*
     Sets the destination stream as a complete copy of the source stream.

     This function can be useful when several compression strategies will be
   tried, for example when there are several ways of pre-processing the input
   data with a filter.  The streams that will be discarded should then be freed
   by calling deflateEnd.  Note that deflateCopy duplicates the internal
   compression state which can be quite large, so this strategy is slow and can
   consume lots of memory.

     deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
   (such as zalloc being Z_NULL).  msg is left unchanged in both source and
   destination.
*/

ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
/*
     This function is equivalent to deflateEnd followed by deflateInit,
   but does not free and reallocate all the internal compression state.  The
   stream will keep the same compression level and any other attributes that
   may have been set by deflateInit2.

     deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent (such as zalloc or state being Z_NULL).
*/

ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
                                      int level,
                                      int strategy));
/*
     Dynamically update the compression level and compression strategy.  The
   interpretation of level and strategy is as in deflateInit2.  This can be
   used to switch between compression and straight copy of the input data, or
   to switch to a different kind of input data requiring a different strategy.
   If the compression level is changed, the input available so far is
   compressed with the old level (and may be flushed); the new level will take
   effect only at the next call of deflate().

     Before the call of deflateParams, the stream state must be set as for
   a call of deflate(), since the currently available input may have to be
   compressed and flushed.  In particular, strm->avail_out must be non-zero.

     deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
   stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if
   strm->avail_out was zero.
*/

ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
                                    int good_length,
                                    int max_lazy,
                                    int nice_length,
                                    int max_chain));
/*
     Fine tune deflate's internal compression parameters.  This should only be
   used by someone who understands the algorithm used by zlib's deflate for
   searching for the best matching string, and even then only by the most
   fanatic optimizer trying to squeeze out the last compressed bit for their
   specific input data.  Read the deflate.c source code for the meaning of the
   max_lazy, good_length, nice_length, and max_chain parameters.

     deflateTune() can be called after deflateInit() or deflateInit2(), and
   returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream.
 */

ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
                                       uLong sourceLen));
/*
     deflateBound() returns an upper bound on the compressed size after
   deflation of sourceLen bytes.  It must be called after deflateInit() or
   deflateInit2(), and after deflateSetHeader(), if used.  This would be used
   to allocate an output buffer for deflation in a single pass, and so would be
   called before deflate().  If that first deflate() call is provided the
   sourceLen input bytes, an output buffer allocated to the size returned by
   deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed
   to return Z_STREAM_END.  Note that it is possible for the compressed size to
   be larger than the value returned by deflateBound() if flush options other
   than Z_FINISH or Z_NO_FLUSH are used.
*/

ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm,
                                       unsigned *pending,
                                       int *bits));
/*
     deflatePending() returns the number of bytes and bits of output that have
   been generated, but not yet provided in the available output.  The bytes not
   provided would be due to the available output space having being consumed.
   The number of bits of output not provided are between 0 and 7, where they
   await more bits to join them in order to fill out a full byte.  If pending
   or bits are Z_NULL, then those values are not set.

     deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent.
 */

ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
                                     int bits,
                                     int value));
/*
     deflatePrime() inserts bits in the deflate output stream.  The intent
   is that this function is used to start off the deflate output with the bits
   leftover from a previous deflate stream when appending to it.  As such, this
   function can only be used for raw deflate, and must be used before the first
   deflate() call after a deflateInit2() or deflateReset().  bits must be less
   than or equal to 16, and that many of the least significant bits of value
   will be inserted in the output.

     deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough
   room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the
   source stream state was inconsistent.
*/

ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
                                         gz_headerp head));
/*
     deflateSetHeader() provides gzip header information for when a gzip
   stream is requested by deflateInit2().  deflateSetHeader() may be called
   after deflateInit2() or deflateReset() and before the first call of
   deflate().  The text, time, os, extra field, name, and comment information
   in the provided gz_header structure are written to the gzip header (xflag is
   ignored -- the extra flags are set according to the compression level).  The
   caller must assure that, if not Z_NULL, name and comment are terminated with
   a zero byte, and that if extra is not Z_NULL, that extra_len bytes are
   available there.  If hcrc is true, a gzip header crc is included.  Note that
   the current versions of the command-line version of gzip (up through version
   1.3.x) do not support header crc's, and will report that it is a "multi-part
   gzip file" and give up.

     If deflateSetHeader is not used, the default gzip header has text false,
   the time set to zero, and os set to 255, with no extra, name, or comment
   fields.  The gzip header is returned to the default state by deflateReset().

     deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent.
*/

/*
ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
                                     int  windowBits));

     This is another version of inflateInit with an extra parameter.  The
   fields next_in, avail_in, zalloc, zfree and opaque must be initialized
   before by the caller.

     The windowBits parameter is the base two logarithm of the maximum window
   size (the size of the history buffer).  It should be in the range 8..15 for
   this version of the library.  The default value is 15 if inflateInit is used
   instead.  windowBits must be greater than or equal to the windowBits value
   provided to deflateInit2() while compressing, or it must be equal to 15 if
   deflateInit2() was not used.  If a compressed stream with a larger window
   size is given as input, inflate() will return with the error code
   Z_DATA_ERROR instead of trying to allocate a larger window.

     windowBits can also be zero to request that inflate use the window size in
   the zlib header of the compressed stream.

     windowBits can also be -8..-15 for raw inflate.  In this case, -windowBits
   determines the window size.  inflate() will then process raw deflate data,
   not looking for a zlib or gzip header, not generating a check value, and not
   looking for any check values for comparison at the end of the stream.  This
   is for use with other formats that use the deflate compressed data format
   such as zip.  Those formats provide their own check values.  If a custom
   format is developed using the raw deflate format for compressed data, it is
   recommended that a check value such as an adler32 or a crc32 be applied to
   the uncompressed data as is done in the zlib, gzip, and zip formats.  For
   most applications, the zlib format should be used as is.  Note that comments
   above on the use in deflateInit2() applies to the magnitude of windowBits.

     windowBits can also be greater than 15 for optional gzip decoding.  Add
   32 to windowBits to enable zlib and gzip decoding with automatic header
   detection, or add 16 to decode only the gzip format (the zlib format will
   return a Z_DATA_ERROR).  If a gzip stream is being decoded, strm->adler is a
   crc32 instead of an adler32.

     inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
   version assumed by the caller, or Z_STREAM_ERROR if the parameters are
   invalid, such as a null pointer to the structure.  msg is set to null if
   there is no error message.  inflateInit2 does not perform any decompression
   apart from possibly reading the zlib header if present: actual decompression
   will be done by inflate().  (So next_in and avail_in may be modified, but
   next_out and avail_out are unused and unchanged.) The current implementation
   of inflateInit2() does not process any header information -- that is
   deferred until inflate() is called.
*/

ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
                                             const Bytef *dictionary,
                                             uInt  dictLength));
/*
     Initializes the decompression dictionary from the given uncompressed byte
   sequence.  This function must be called immediately after a call of inflate,
   if that call returned Z_NEED_DICT.  The dictionary chosen by the compressor
   can be determined from the adler32 value returned by that call of inflate.
   The compressor and decompressor must use exactly the same dictionary (see
   deflateSetDictionary).  For raw inflate, this function can be called at any
   time to set the dictionary.  If the provided dictionary is smaller than the
   window and there is already data in the window, then the provided dictionary
   will amend what's there.  The application must insure that the dictionary
   that was used for compression is provided.

     inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
   parameter is invalid (e.g.  dictionary being Z_NULL) or the stream state is
   inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
   expected one (incorrect adler32 value).  inflateSetDictionary does not
   perform any decompression: this will be done by subsequent calls of
   inflate().
*/

ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm,
                                             Bytef *dictionary,
                                             uInt  *dictLength));
/*
     Returns the sliding dictionary being maintained by inflate.  dictLength is
   set to the number of bytes in the dictionary, and that many bytes are copied
   to dictionary.  dictionary must have enough space, where 32768 bytes is
   always enough.  If inflateGetDictionary() is called with dictionary equal to
   Z_NULL, then only the dictionary length is returned, and nothing is copied.
   Similary, if dictLength is Z_NULL, then it is not set.

     inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the
   stream state is inconsistent.
*/

ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
/*
     Skips invalid compressed data until a possible full flush point (see above
   for the description of deflate with Z_FULL_FLUSH) can be found, or until all
   available input is skipped.  No output is provided.

     inflateSync searches for a 00 00 FF FF pattern in the compressed data.
   All full flush points have this pattern, but not all occurrences of this
   pattern are full flush points.

     inflateSync returns Z_OK if a possible full flush point has been found,
   Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point
   has been found, or Z_STREAM_ERROR if the stream structure was inconsistent.
   In the success case, the application may save the current current value of
   total_in which indicates where valid compressed data was found.  In the
   error case, the application may repeatedly call inflateSync, providing more
   input each time, until success or end of the input data.
*/

ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
                                    z_streamp source));
/*
     Sets the destination stream as a complete copy of the source stream.

     This function can be useful when randomly accessing a large stream.  The
   first pass through the stream can periodically record the inflate state,
   allowing restarting inflate at those points when randomly accessing the
   stream.

     inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
   (such as zalloc being Z_NULL).  msg is left unchanged in both source and
   destination.
*/

ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
/*
     This function is equivalent to inflateEnd followed by inflateInit,
   but does not free and reallocate all the internal decompression state.  The
   stream will keep attributes that may have been set by inflateInit2.

     inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent (such as zalloc or state being Z_NULL).
*/

ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm,
                                      int windowBits));
/*
     This function is the same as inflateReset, but it also permits changing
   the wrap and window size requests.  The windowBits parameter is interpreted
   the same as it is for inflateInit2.

     inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent (such as zalloc or state being Z_NULL), or if
   the windowBits parameter is invalid.
*/

ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
                                     int bits,
                                     int value));
/*
     This function inserts bits in the inflate input stream.  The intent is
   that this function is used to start inflating at a bit position in the
   middle of a byte.  The provided bits will be used before any bytes are used
   from next_in.  This function should only be used with raw inflate, and
   should be used before the first inflate() call after inflateInit2() or
   inflateReset().  bits must be less than or equal to 16, and that many of the
   least significant bits of value will be inserted in the input.

     If bits is negative, then the input stream bit buffer is emptied.  Then
   inflatePrime() can be called again to put bits in the buffer.  This is used
   to clear out bits leftover after feeding inflate a block description prior
   to feeding inflate codes.

     inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent.
*/

ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm));
/*
     This function returns two values, one in the lower 16 bits of the return
   value, and the other in the remaining upper bits, obtained by shifting the
   return value down 16 bits.  If the upper value is -1 and the lower value is
   zero, then inflate() is currently decoding information outside of a block.
   If the upper value is -1 and the lower value is non-zero, then inflate is in
   the middle of a stored block, with the lower value equaling the number of
   bytes from the input remaining to copy.  If the upper value is not -1, then
   it is the number of bits back from the current bit position in the input of
   the code (literal or length/distance pair) currently being processed.  In
   that case the lower value is the number of bytes already emitted for that
   code.

     A code is being processed if inflate is waiting for more input to complete
   decoding of the code, or if it has completed decoding but is waiting for
   more output space to write the literal or match data.

     inflateMark() is used to mark locations in the input data for random
   access, which may be at bit positions, and to note those cases where the
   output of a code may span boundaries of random access blocks.  The current
   location in the input stream can be determined from avail_in and data_type
   as noted in the description for the Z_BLOCK flush parameter for inflate.

     inflateMark returns the value noted above or -1 << 16 if the provided
   source stream state was inconsistent.
*/

ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
                                         gz_headerp head));
/*
     inflateGetHeader() requests that gzip header information be stored in the
   provided gz_header structure.  inflateGetHeader() may be called after
   inflateInit2() or inflateReset(), and before the first call of inflate().
   As inflate() processes the gzip stream, head->done is zero until the header
   is completed, at which time head->done is set to one.  If a zlib stream is
   being decoded, then head->done is set to -1 to indicate that there will be
   no gzip header information forthcoming.  Note that Z_BLOCK or Z_TREES can be
   used to force inflate() to return immediately after header processing is
   complete and before any actual data is decompressed.

     The text, time, xflags, and os fields are filled in with the gzip header
   contents.  hcrc is set to true if there is a header CRC.  (The header CRC
   was valid if done is set to one.) If extra is not Z_NULL, then extra_max
   contains the maximum number of bytes to write to extra.  Once done is true,
   extra_len contains the actual extra field length, and extra contains the
   extra field, or that field truncated if extra_max is less than extra_len.
   If name is not Z_NULL, then up to name_max characters are written there,
   terminated with a zero unless the length is greater than name_max.  If
   comment is not Z_NULL, then up to comm_max characters are written there,
   terminated with a zero unless the length is greater than comm_max.  When any
   of extra, name, or comment are not Z_NULL and the respective field is not
   present in the header, then that field is set to Z_NULL to signal its
   absence.  This allows the use of deflateSetHeader() with the returned
   structure to duplicate the header.  However if those fields are set to
   allocated memory, then the application will need to save those pointers
   elsewhere so that they can be eventually freed.

     If inflateGetHeader is not used, then the header information is simply
   discarded.  The header is always checked for validity, including the header
   CRC if present.  inflateReset() will reset the process to discard the header
   information.  The application would need to call inflateGetHeader() again to
   retrieve the header from the next gzip stream.

     inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
   stream state was inconsistent.
*/

/*
ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
                                        unsigned char FAR *window));

     Initialize the internal stream state for decompression using inflateBack()
   calls.  The fields zalloc, zfree and opaque in strm must be initialized
   before the call.  If zalloc and zfree are Z_NULL, then the default library-
   derived memory allocation routines are used.  windowBits is the base two
   logarithm of the window size, in the range 8..15.  window is a caller
   supplied buffer of that size.  Except for special applications where it is
   assured that deflate was used with small window sizes, windowBits must be 15
   and a 32K byte window must be supplied to be able to decompress general
   deflate streams.

     See inflateBack() for the usage of these routines.

     inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of
   the parameters are invalid, Z_MEM_ERROR if the internal state could not be
   allocated, or Z_VERSION_ERROR if the version of the library does not match
   the version of the header file.
*/

typedef unsigned (*in_func) OF((void FAR *,
                                z_const unsigned char FAR * FAR *));
typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned));

ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
                                    in_func in, void FAR *in_desc,
                                    out_func out, void FAR *out_desc));
/*
     inflateBack() does a raw inflate with a single call using a call-back
   interface for input and output.  This is potentially more efficient than
   inflate() for file i/o applications, in that it avoids copying between the
   output and the sliding window by simply making the window itself the output
   buffer.  inflate() can be faster on modern CPUs when used with large
   buffers.  inflateBack() trusts the application to not change the output
   buffer passed by the output function, at least until inflateBack() returns.

     inflateBackInit() must be called first to allocate the internal state
   and to initialize the state with the user-provided window buffer.
   inflateBack() may then be used multiple times to inflate a complete, raw
   deflate stream with each call.  inflateBackEnd() is then called to free the
   allocated state.

     A raw deflate stream is one with no zlib or gzip header or trailer.
   This routine would normally be used in a utility that reads zip or gzip
   files and writes out uncompressed files.  The utility would decode the
   header and process the trailer on its own, hence this routine expects only
   the raw deflate stream to decompress.  This is different from the normal
   behavior of inflate(), which expects either a zlib or gzip header and
   trailer around the deflate stream.

     inflateBack() uses two subroutines supplied by the caller that are then
   called by inflateBack() for input and output.  inflateBack() calls those
   routines until it reads a complete deflate stream and writes out all of the
   uncompressed data, or until it encounters an error.  The function's
   parameters and return types are defined above in the in_func and out_func
   typedefs.  inflateBack() will call in(in_desc, &buf) which should return the
   number of bytes of provided input, and a pointer to that input in buf.  If
   there is no input available, in() must return zero--buf is ignored in that
   case--and inflateBack() will return a buffer error.  inflateBack() will call
   out(out_desc, buf, len) to write the uncompressed data buf[0..len-1].  out()
   should return zero on success, or non-zero on failure.  If out() returns
   non-zero, inflateBack() will return with an error.  Neither in() nor out()
   are permitted to change the contents of the window provided to
   inflateBackInit(), which is also the buffer that out() uses to write from.
   The length written by out() will be at most the window size.  Any non-zero
   amount of input may be provided by in().

     For convenience, inflateBack() can be provided input on the first call by
   setting strm->next_in and strm->avail_in.  If that input is exhausted, then
   in() will be called.  Therefore strm->next_in must be initialized before
   calling inflateBack().  If strm->next_in is Z_NULL, then in() will be called
   immediately for input.  If strm->next_in is not Z_NULL, then strm->avail_in
   must also be initialized, and then if strm->avail_in is not zero, input will
   initially be taken from strm->next_in[0 ..  strm->avail_in - 1].

     The in_desc and out_desc parameters of inflateBack() is passed as the
   first parameter of in() and out() respectively when they are called.  These
   descriptors can be optionally used to pass any information that the caller-
   supplied in() and out() functions need to do their job.

     On return, inflateBack() will set strm->next_in and strm->avail_in to
   pass back any unused input that was provided by the last in() call.  The
   return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR
   if in() or out() returned an error, Z_DATA_ERROR if there was a format error
   in the deflate stream (in which case strm->msg is set to indicate the nature
   of the error), or Z_STREAM_ERROR if the stream was not properly initialized.
   In the case of Z_BUF_ERROR, an input or output error can be distinguished
   using strm->next_in which will be Z_NULL only if in() returned an error.  If
   strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning
   non-zero.  (in() will always be called before out(), so strm->next_in is
   assured to be defined if out() returns non-zero.) Note that inflateBack()
   cannot return Z_OK.
*/

ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
/*
     All memory allocated by inflateBackInit() is freed.

     inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream
   state was inconsistent.
*/

ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
/* Return flags indicating compile-time options.

    Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other:
     1.0: size of uInt
     3.2: size of uLong
     5.4: size of voidpf (pointer)
     7.6: size of z_off_t

    Compiler, assembler, and debug options:
     8: DEBUG
     9: ASMV or ASMINF -- use ASM code
     10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention
     11: 0 (reserved)

    One-time table building (smaller code, but not thread-safe if true):
     12: BUILDFIXED -- build static block decoding tables when needed
     13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed
     14,15: 0 (reserved)

    Library content (indicates missing functionality):
     16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking
                          deflate code when not needed)
     17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect
                    and decode gzip streams (to avoid linking crc code)
     18-19: 0 (reserved)

    Operation variations (changes in library functionality):
     20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate
     21: FASTEST -- deflate algorithm with only one, lowest compression level
     22,23: 0 (reserved)

    The sprintf variant used by gzprintf (zero is best):
     24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format
     25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure!
     26: 0 = returns value, 1 = void -- 1 means inferred string length returned

    Remainder:
     27-31: 0 (reserved)
 */

#ifndef Z_SOLO

                        /* utility functions */

/*
     The following utility functions are implemented on top of the basic
   stream-oriented functions.  To simplify the interface, some default options
   are assumed (compression level and memory usage, standard memory allocation
   functions).  The source code of these utility functions can be modified if
   you need special options.
*/

ZEXTERN int ZEXPORT compress OF((Bytef *dest,   uLongf *destLen,
                                 const Bytef *source, uLong sourceLen));
/*
     Compresses the source buffer into the destination buffer.  sourceLen is
   the byte length of the source buffer.  Upon entry, destLen is the total size
   of the destination buffer, which must be at least the value returned by
   compressBound(sourceLen).  Upon exit, destLen is the actual size of the
   compressed buffer.

     compress returns Z_OK if success, Z_MEM_ERROR if there was not
   enough memory, Z_BUF_ERROR if there was not enough room in the output
   buffer.
*/

ZEXTERN int ZEXPORT compress2 OF((Bytef *dest,   uLongf *destLen,
                                  const Bytef *source, uLong sourceLen,
                                  int level));
/*
     Compresses the source buffer into the destination buffer.  The level
   parameter has the same meaning as in deflateInit.  sourceLen is the byte
   length of the source buffer.  Upon entry, destLen is the total size of the
   destination buffer, which must be at least the value returned by
   compressBound(sourceLen).  Upon exit, destLen is the actual size of the
   compressed buffer.

     compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
   memory, Z_BUF_ERROR if there was not enough room in the output buffer,
   Z_STREAM_ERROR if the level parameter is invalid.
*/

ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen));
/*
     compressBound() returns an upper bound on the compressed size after
   compress() or compress2() on sourceLen bytes.  It would be used before a
   compress() or compress2() call to allocate the destination buffer.
*/

ZEXTERN int ZEXPORT uncompress OF((Bytef *dest,   uLongf *destLen,
                                   const Bytef *source, uLong sourceLen));
/*
     Decompresses the source buffer into the destination buffer.  sourceLen is
   the byte length of the source buffer.  Upon entry, destLen is the total size
   of the destination buffer, which must be large enough to hold the entire
   uncompressed data.  (The size of the uncompressed data must have been saved
   previously by the compressor and transmitted to the decompressor by some
   mechanism outside the scope of this compression library.) Upon exit, destLen
   is the actual size of the uncompressed buffer.

     uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
   enough memory, Z_BUF_ERROR if there was not enough room in the output
   buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete.  In
   the case where there is not enough room, uncompress() will fill the output
   buffer with the uncompressed data up to that point.
*/

                        /* gzip file access functions */

/*
     This library supports reading and writing files in gzip (.gz) format with
   an interface similar to that of stdio, using the functions that start with
   "gz".  The gzip format is different from the zlib format.  gzip is a gzip
   wrapper, documented in RFC 1952, wrapped around a deflate stream.
*/

typedef struct gzFile_s *gzFile;    /* semi-opaque gzip file descriptor */

/*
ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));

     Opens a gzip (.gz) file for reading or writing.  The mode parameter is as
   in fopen ("rb" or "wb") but can also include a compression level ("wb9") or
   a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only
   compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F'
   for fixed code compression as in "wb9F".  (See the description of
   deflateInit2 for more information about the strategy parameter.)  'T' will
   request transparent writing or appending with no compression and not using
   the gzip format.

     "a" can be used instead of "w" to request that the gzip stream that will
   be written be appended to the file.  "+" will result in an error, since
   reading and writing to the same gzip file is not supported.  The addition of
   "x" when writing will create the file exclusively, which fails if the file
   already exists.  On systems that support it, the addition of "e" when
   reading or writing will set the flag to close the file on an execve() call.

     These functions, as well as gzip, will read and decode a sequence of gzip
   streams in a file.  The append function of gzopen() can be used to create
   such a file.  (Also see gzflush() for another way to do this.)  When
   appending, gzopen does not test whether the file begins with a gzip stream,
   nor does it look for the end of the gzip streams to begin appending.  gzopen
   will simply append a gzip stream to the existing file.

     gzopen can be used to read a file which is not in gzip format; in this
   case gzread will directly read from the file without decompression.  When
   reading, this will be detected automatically by looking for the magic two-
   byte gzip header.

     gzopen returns NULL if the file could not be opened, if there was
   insufficient memory to allocate the gzFile state, or if an invalid mode was
   specified (an 'r', 'w', or 'a' was not provided, or '+' was provided).
   errno can be checked to determine if the reason gzopen failed was that the
   file could not be opened.
*/

ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
/*
     gzdopen associates a gzFile with the file descriptor fd.  File descriptors
   are obtained from calls like open, dup, creat, pipe or fileno (if the file
   has been previously opened with fopen).  The mode parameter is as in gzopen.

     The next call of gzclose on the returned gzFile will also close the file
   descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor
   fd.  If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd,
   mode);.  The duplicated descriptor should be saved to avoid a leak, since
   gzdopen does not close fd if it fails.  If you are using fileno() to get the
   file descriptor from a FILE *, then you will have to use dup() to avoid
   double-close()ing the file descriptor.  Both gzclose() and fclose() will
   close the associated file descriptor, so they need to have different file
   descriptors.

     gzdopen returns NULL if there was insufficient memory to allocate the
   gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not
   provided, or '+' was provided), or if fd is -1.  The file descriptor is not
   used until the next gz* read, write, seek, or close operation, so gzdopen
   will not detect if fd is invalid (unless fd is -1).
*/

ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size));
/*
     Set the internal buffer size used by this library's functions.  The
   default buffer size is 8192 bytes.  This function must be called after
   gzopen() or gzdopen(), and before any other calls that read or write the
   file.  The buffer memory allocation is always deferred to the first read or
   write.  Two buffers are allocated, either both of the specified size when
   writing, or one of the specified size and the other twice that size when
   reading.  A larger buffer size of, for example, 64K or 128K bytes will
   noticeably increase the speed of decompression (reading).

     The new buffer size also affects the maximum length for gzprintf().

     gzbuffer() returns 0 on success, or -1 on failure, such as being called
   too late.
*/

ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
/*
     Dynamically update the compression level or strategy.  See the description
   of deflateInit2 for the meaning of these parameters.

     gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not
   opened for writing.
*/

ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
/*
     Reads the given number of uncompressed bytes from the compressed file.  If
   the input file is not in gzip format, gzread copies the given number of
   bytes into the buffer directly from the file.

     After reaching the end of a gzip stream in the input, gzread will continue
   to read, looking for another gzip stream.  Any number of gzip streams may be
   concatenated in the input file, and will all be decompressed by gzread().
   If something other than a gzip stream is encountered after a gzip stream,
   that remaining trailing garbage is ignored (and no error is returned).

     gzread can be used to read a gzip file that is being concurrently written.
   Upon reaching the end of the input, gzread will return with the available
   data.  If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then
   gzclearerr can be used to clear the end of file indicator in order to permit
   gzread to be tried again.  Z_OK indicates that a gzip stream was completed
   on the last gzread.  Z_BUF_ERROR indicates that the input file ended in the
   middle of a gzip stream.  Note that gzread does not return -1 in the event
   of an incomplete gzip stream.  This error is deferred until gzclose(), which
   will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip
   stream.  Alternatively, gzerror can be used before gzclose to detect this
   case.

     gzread returns the number of uncompressed bytes actually read, less than
   len for end of file, or -1 for error.
*/

ZEXTERN int ZEXPORT gzwrite OF((gzFile file,
                                voidpc buf, unsigned len));
/*
     Writes the given number of uncompressed bytes into the compressed file.
   gzwrite returns the number of uncompressed bytes written or 0 in case of
   error.
*/

ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...));
/*
     Converts, formats, and writes the arguments to the compressed file under
   control of the format string, as in fprintf.  gzprintf returns the number of
   uncompressed bytes actually written, or 0 in case of error.  The number of
   uncompressed bytes written is limited to 8191, or one less than the buffer
   size given to gzbuffer().  The caller should assure that this limit is not
   exceeded.  If it is exceeded, then gzprintf() will return an error (0) with
   nothing written.  In this case, there may also be a buffer overflow with
   unpredictable consequences, which is possible only if zlib was compiled with
   the insecure functions sprintf() or vsprintf() because the secure snprintf()
   or vsnprintf() functions were not available.  This can be determined using
   zlibCompileFlags().
*/

ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
/*
     Writes the given null-terminated string to the compressed file, excluding
   the terminating null character.

     gzputs returns the number of characters written, or -1 in case of error.
*/

ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
/*
     Reads bytes from the compressed file until len-1 characters are read, or a
   newline character is read and transferred to buf, or an end-of-file
   condition is encountered.  If any characters are read or if len == 1, the
   string is terminated with a null character.  If no characters are read due
   to an end-of-file or len < 1, then the buffer is left untouched.

     gzgets returns buf which is a null-terminated string, or it returns NULL
   for end-of-file or in case of error.  If there was an error, the contents at
   buf are indeterminate.
*/

ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c));
/*
     Writes c, converted to an unsigned char, into the compressed file.  gzputc
   returns the value that was written, or -1 in case of error.
*/

ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
/*
     Reads one byte from the compressed file.  gzgetc returns this byte or -1
   in case of end of file or error.  This is implemented as a macro for speed.
   As such, it does not do all of the checking the other functions do.  I.e.
   it does not check to see if file is NULL, nor whether the structure file
   points to has been clobbered or not.
*/

ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
/*
     Push one character back onto the stream to be read as the first character
   on the next read.  At least one character of push-back is allowed.
   gzungetc() returns the character pushed, or -1 on failure.  gzungetc() will
   fail if c is -1, and may fail if a character has been pushed but not read
   yet.  If gzungetc is used immediately after gzopen or gzdopen, at least the
   output buffer size of pushed characters is allowed.  (See gzbuffer above.)
   The pushed character will be discarded if the stream is repositioned with
   gzseek() or gzrewind().
*/

ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
/*
     Flushes all pending output into the compressed file.  The parameter flush
   is as in the deflate() function.  The return value is the zlib error number
   (see function gzerror below).  gzflush is only permitted when writing.

     If the flush parameter is Z_FINISH, the remaining data is written and the
   gzip stream is completed in the output.  If gzwrite() is called again, a new
   gzip stream will be started in the output.  gzread() is able to read such
   concatented gzip streams.

     gzflush should be called only when strictly necessary because it will
   degrade compression if called too often.
*/

/*
ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
                                   z_off_t offset, int whence));

     Sets the starting position for the next gzread or gzwrite on the given
   compressed file.  The offset represents a number of bytes in the
   uncompressed data stream.  The whence parameter is defined as in lseek(2);
   the value SEEK_END is not supported.

     If the file is opened for reading, this function is emulated but can be
   extremely slow.  If the file is opened for writing, only forward seeks are
   supported; gzseek then compresses a sequence of zeroes up to the new
   starting position.

     gzseek returns the resulting offset location as measured in bytes from
   the beginning of the uncompressed stream, or -1 in case of error, in
   particular if the file is opened for writing and the new starting position
   would be before the current position.
*/

ZEXTERN int ZEXPORT    gzrewind OF((gzFile file));
/*
     Rewinds the given file. This function is supported only for reading.

     gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
*/

/*
ZEXTERN z_off_t ZEXPORT    gztell OF((gzFile file));

     Returns the starting position for the next gzread or gzwrite on the given
   compressed file.  This position represents a number of bytes in the
   uncompressed data stream, and is zero when starting, even if appending or
   reading a gzip stream from the middle of a file using gzdopen().

     gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
*/

/*
ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file));

     Returns the current offset in the file being read or written.  This offset
   includes the count of bytes that precede the gzip stream, for example when
   appending or when using gzdopen() for reading.  When reading, the offset
   does not include as yet unused buffered input.  This information can be used
   for a progress indicator.  On error, gzoffset() returns -1.
*/

ZEXTERN int ZEXPORT gzeof OF((gzFile file));
/*
     Returns true (1) if the end-of-file indicator has been set while reading,
   false (0) otherwise.  Note that the end-of-file indicator is set only if the
   read tried to go past the end of the input, but came up short.  Therefore,
   just like feof(), gzeof() may return false even if there is no more data to
   read, in the event that the last read request was for the exact number of
   bytes remaining in the input file.  This will happen if the input file size
   is an exact multiple of the buffer size.

     If gzeof() returns true, then the read functions will return no more data,
   unless the end-of-file indicator is reset by gzclearerr() and the input file
   has grown since the previous end of file was detected.
*/

ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
/*
     Returns true (1) if file is being copied directly while reading, or false
   (0) if file is a gzip stream being decompressed.

     If the input file is empty, gzdirect() will return true, since the input
   does not contain a gzip stream.

     If gzdirect() is used immediately after gzopen() or gzdopen() it will
   cause buffers to be allocated to allow reading the file to determine if it
   is a gzip file.  Therefore if gzbuffer() is used, it should be called before
   gzdirect().

     When writing, gzdirect() returns true (1) if transparent writing was
   requested ("wT" for the gzopen() mode), or false (0) otherwise.  (Note:
   gzdirect() is not needed when writing.  Transparent writing must be
   explicitly requested, so the application already knows the answer.  When
   linking statically, using gzdirect() will include all of the zlib code for
   gzip file reading and decompression, which may not be desired.)
*/

ZEXTERN int ZEXPORT    gzclose OF((gzFile file));
/*
     Flushes all pending output if necessary, closes the compressed file and
   deallocates the (de)compression state.  Note that once file is closed, you
   cannot call gzerror with file, since its structures have been deallocated.
   gzclose must not be called more than once on the same file, just as free
   must not be called more than once on the same allocation.

     gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a
   file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the
   last read ended in the middle of a gzip stream, or Z_OK on success.
*/

ZEXTERN int ZEXPORT gzclose_r OF((gzFile file));
ZEXTERN int ZEXPORT gzclose_w OF((gzFile file));
/*
     Same as gzclose(), but gzclose_r() is only for use when reading, and
   gzclose_w() is only for use when writing or appending.  The advantage to
   using these instead of gzclose() is that they avoid linking in zlib
   compression or decompression code that is not used when only reading or only
   writing respectively.  If gzclose() is used, then both compression and
   decompression code will be included the application when linking to a static
   zlib library.
*/

ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
/*
     Returns the error message for the last error which occurred on the given
   compressed file.  errnum is set to zlib error number.  If an error occurred
   in the file system and not in the compression library, errnum is set to
   Z_ERRNO and the application may consult errno to get the exact error code.

     The application must not modify the returned string.  Future calls to
   this function may invalidate the previously returned string.  If file is
   closed, then the string previously returned by gzerror will no longer be
   available.

     gzerror() should be used to distinguish errors from end-of-file for those
   functions above that do not distinguish those cases in their return values.
*/

ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
/*
     Clears the error and end-of-file flags for file.  This is analogous to the
   clearerr() function in stdio.  This is useful for continuing to read a gzip
   file that is being written concurrently.
*/

#endif /* !Z_SOLO */
# 1560 "/usr/include/zlib.h" 3 4

                        /* checksum functions */

/*
     These functions are not related to compression but are exported
   anyway because they might be useful in applications using the compression
   library.
*/

ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
/*
     Update a running Adler-32 checksum with the bytes buf[0..len-1] and
   return the updated checksum.  If buf is Z_NULL, this function returns the
   required initial value for the checksum.

     An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
   much faster.

   Usage example:

     uLong adler = adler32(0L, Z_NULL, 0);

     while (read_buffer(buffer, length) != EOF) {
       adler = adler32(adler, buffer, length);
     }
     if (adler != original_adler) error();
*/

/*
ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
                                          z_off_t len2));

     Combine two Adler-32 checksums into one.  For two sequences of bytes, seq1
   and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for
   each, adler1 and adler2.  adler32_combine() returns the Adler-32 checksum of
   seq1 and seq2 concatenated, requiring only adler1, adler2, and len2.  Note
   that the z_off_t type (like off_t) is a signed integer.  If len2 is
   negative, the result has no meaning or utility.
*/

ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
/*
     Update a running CRC-32 with the bytes buf[0..len-1] and return the
   updated CRC-32.  If buf is Z_NULL, this function returns the required
   initial value for the crc.  Pre- and post-conditioning (one's complement) is
   performed within this function so it shouldn't be done by the application.

   Usage example:

     uLong crc = crc32(0L, Z_NULL, 0);

     while (read_buffer(buffer, length) != EOF) {
       crc = crc32(crc, buffer, length);
     }
     if (crc != original_crc) error();
*/

/*
ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));

     Combine two CRC-32 check values into one.  For two sequences of bytes,
   seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
   calculated for each, crc1 and crc2.  crc32_combine() returns the CRC-32
   check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
   len2.
*/


                        /* various hacks, don't look :) */

/* deflateInit and inflateInit are macros to allow checking the zlib version
 * and the compiler's view of z_stream:
 */
ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
                                     const char *version, int stream_size));
ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
                                     const char *version, int stream_size));
ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int  level, int  method,
                                      int windowBits, int memLevel,
                                      int strategy, const char *version,
                                      int stream_size));
ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int  windowBits,
                                      const char *version, int stream_size));
ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
                                         unsigned char FAR *window,
                                         const char *version,
                                         int stream_size));
#define deflateInit(strm, level) \
        deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream))
#define inflateInit(strm) \
        inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream))
#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
        deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
                      (strategy), ZLIB_VERSION, (int)sizeof(z_stream))
#define inflateInit2(strm, windowBits) \
        inflateInit2_((strm), (windowBits), ZLIB_VERSION, \
                      (int)sizeof(z_stream))
#define inflateBackInit(strm, windowBits, window) \
        inflateBackInit_((strm), (windowBits), (window), \
                      ZLIB_VERSION, (int)sizeof(z_stream))

#ifndef Z_SOLO

/* gzgetc() macro and its supporting function and exposed data structure.  Note
 * that the real internal state is much larger than the exposed structure.
 * This abbreviated structure exposes just enough for the gzgetc() macro.  The
 * user should not mess with these exposed elements, since their names or
 * behavior could change in the future, perhaps even capriciously.  They can
 * only be used by the gzgetc() macro.  You have been warned.
 */
struct gzFile_s {
    unsigned have;
    unsigned char *next;
    z_off64_t pos;
};
ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file));  /* backward compatibility */
#ifdef Z_PREFIX_SET
#  undef z_gzgetc
#  define z_gzgetc(g) \
          ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g))
#else
# 1681 "/usr/include/zlib.h" 3 4
#  define gzgetc(g) \
          ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g))
#endif
# 1684 "/usr/include/zlib.h" 3 4

/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or
 * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if
 * both are true, the application gets the *64 functions, and the regular
 * functions are changed to 64 bits) -- in case these are set on systems
 * without large file support, _LFS64_LARGEFILE must also be true
 */
#ifdef Z_LARGE64
   ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
   ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int));
   ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile));
   ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
   ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
   ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
#endif
# 1699 "/usr/include/zlib.h" 3 4

#if !defined(ZLIB_INTERNAL) && defined(Z_WANT64)
#  ifdef Z_PREFIX_SET
#    define z_gzopen z_gzopen64
#    define z_gzseek z_gzseek64
#    define z_gztell z_gztell64
#    define z_gzoffset z_gzoffset64
#    define z_adler32_combine z_adler32_combine64
#    define z_crc32_combine z_crc32_combine64
#  else
# 1709 "/usr/include/zlib.h" 3 4
#    define gzopen gzopen64
#    define gzseek gzseek64
#    define gztell gztell64
#    define gzoffset gzoffset64
#    define adler32_combine adler32_combine64
#    define crc32_combine crc32_combine64
#  endif
# 1716 "/usr/include/zlib.h" 3 4
#  ifndef Z_LARGE64
     ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
     ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int));
     ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile));
     ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile));
     ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
     ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
#  endif
# 1724 "/usr/include/zlib.h" 3 4
#else
# 1725 "/usr/include/zlib.h" 3 4
   ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *));
   ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int));
   ZEXTERN z_off_t ZEXPORT gztell OF((gzFile));
   ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile));
   ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
   ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
#endif
# 1732 "/usr/include/zlib.h" 3 4

#else /* Z_SOLO */
# 1734 "/usr/include/zlib.h" 3 4

   ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
   ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));

#endif /* !Z_SOLO */
# 1739 "/usr/include/zlib.h" 3 4

/* hack for buggy compilers */
#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL)
    struct internal_state {int dummy;};
#endif
# 1744 "/usr/include/zlib.h" 3 4

/* undocumented functions */
ZEXTERN const char   * ZEXPORT zError           OF((int));
ZEXTERN int            ZEXPORT inflateSyncPoint OF((z_streamp));
ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table    OF((void));
ZEXTERN int            ZEXPORT inflateUndermine OF((z_streamp, int));
ZEXTERN int            ZEXPORT inflateResetKeep OF((z_streamp));
ZEXTERN int            ZEXPORT deflateResetKeep OF((z_streamp));
#if defined(_WIN32) && !defined(Z_SOLO)
ZEXTERN gzFile         ZEXPORT gzopen_w OF((const wchar_t *path,
                                            const char *mode));
#endif
# 1756 "/usr/include/zlib.h" 3 4
#if defined(STDC) || defined(Z_HAVE_STDARG_H)
#  ifndef Z_SOLO
ZEXTERN int            ZEXPORTVA gzvprintf Z_ARG((gzFile file,
                                                  const char *format,
                                                  va_list va));
#  endif
# 1762 "/usr/include/zlib.h" 3 4
#endif
# 1763 "/usr/include/zlib.h" 3 4

#ifdef __cplusplus
}
#endif
# 1767 "/usr/include/zlib.h" 3 4

#endif /* ZLIB_H */
# 1769 "/usr/include/zlib.h" 3 4
# 27 "beam/dist.h" 2

#define DFLAG_PUBLISHED           0x01
#define DFLAG_ATOM_CACHE          0x02
#define DFLAG_EXTENDED_REFERENCES 0x04
#define DFLAG_DIST_MONITOR        0x08
#define DFLAG_FUN_TAGS            0x10
#define DFLAG_DIST_MONITOR_NAME   0x20
#define DFLAG_HIDDEN_ATOM_CACHE   0x40
#define DFLAG_NEW_FUN_TAGS        0x80
#define DFLAG_EXTENDED_PIDS_PORTS 0x100
#define DFLAG_EXPORT_PTR_TAG      0x200
#define DFLAG_BIT_BINARIES        0x400
#define DFLAG_NEW_FLOATS          0x800
#define DFLAG_UNICODE_IO          0x1000
#define DFLAG_DIST_HDR_ATOM_CACHE 0x2000
#define DFLAG_SMALL_ATOM_TAGS     0x4000
#define DFLAG_INTERNAL_TAGS       0x8000
#define DFLAG_UTF8_ATOMS          0x10000
#define DFLAG_MAP_TAG             0x20000

/* All flags that should be enabled when term_to_binary/1 is used. */
#define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES	\
			       | DFLAG_NEW_FUN_TAGS		\
			       | DFLAG_NEW_FLOATS		\
			       | DFLAG_EXTENDED_PIDS_PORTS	\
			       | DFLAG_EXPORT_PTR_TAG		\
			       | DFLAG_BIT_BINARIES             \
			       | DFLAG_MAP_TAG)

/* opcodes used in distribution messages */
#define DOP_LINK		1
#define DOP_SEND		2
#define DOP_EXIT		3
#define DOP_UNLINK		4
/* Ancient DOP_NODE_LINK (5) was here, can be reused */
#define DOP_REG_SEND		6
#define DOP_GROUP_LEADER	7
#define DOP_EXIT2		8

#define DOP_SEND_TT		12
#define DOP_EXIT_TT		13
#define DOP_REG_SEND_TT		16
#define DOP_EXIT2_TT		18

#define DOP_MONITOR_P		19
#define DOP_DEMONITOR_P		20
#define DOP_MONITOR_P_EXIT	21

/* distribution trap functions */
extern Export* dsend2_trap;
extern Export* dsend3_trap;
extern Export* dlink_trap;
extern Export* dunlink_trap;
extern Export* dmonitor_node_trap;
extern Export* dgroup_leader_trap;
extern Export* dexit_trap;
extern Export* dmonitor_p_trap;

typedef enum {
    ERTS_DSP_NO_LOCK,
    ERTS_DSP_RLOCK,
    ERTS_DSP_RWLOCK
} ErtsDSigPrepLock;


typedef struct {
    Process *proc;
    DistEntry *dep;
    Eterm cid;
    Eterm connection_id;
    int no_suspend;
} ErtsDSigData;

#define ERTS_DE_IS_NOT_CONNECTED(DEP) \
  (ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&(DEP)->rwmtx) \
		      || erts_lc_rwmtx_is_rwlocked(&(DEP)->rwmtx)), \
   (is_nil((DEP)->cid) || ((DEP)->status & ERTS_DE_SFLG_EXITING)))

#define ERTS_DE_IS_CONNECTED(DEP) \
  (!ERTS_DE_IS_NOT_CONNECTED((DEP)))

#define ERTS_DE_BUSY_LIMIT (1024*1024)
extern int erts_dist_buf_busy_limit;
extern int erts_is_alive;

/*
 * erts_dsig_prepare() prepares a send of a distributed signal.
 * One of the values defined below are returned. If the returned
 * value is another than ERTS_DSIG_PREP_CONNECTED, the
 * distributed signal cannot be sent before apropriate actions
 * have been taken. Apropriate actions would typically be setting
 * up the connection.
 */

/* Connected; signal can be sent. */
#define ERTS_DSIG_PREP_CONNECTED	0
/* Not connected; connection needs to be set up. */
#define ERTS_DSIG_PREP_NOT_CONNECTED	1
/* Caller would be suspended on send operation. */
#define ERTS_DSIG_PREP_WOULD_SUSPEND	2
/* System not alive (distributed) */
#define ERTS_DSIG_PREP_NOT_ALIVE	3

ERTS_GLB_INLINE int erts_dsig_prepare(ErtsDSigData *,
				      DistEntry *,
				      Process *,
				      ErtsDSigPrepLock,
				      int);

ERTS_GLB_INLINE
void erts_schedule_dist_command(Port *, DistEntry *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE int 
erts_dsig_prepare(ErtsDSigData *dsdp,
		  DistEntry *dep,
		  Process *proc,
		  ErtsDSigPrepLock dspl,
		  int no_suspend)
{
    int failure;
    if (!erts_is_alive)
	return ERTS_DSIG_PREP_NOT_ALIVE;
    if (!dep)
	return ERTS_DSIG_PREP_NOT_CONNECTED;
    if (dspl == ERTS_DSP_RWLOCK)
	erts_smp_de_rwlock(dep);
    else
	erts_smp_de_rlock(dep);
    if (ERTS_DE_IS_NOT_CONNECTED(dep)) {
	failure = ERTS_DSIG_PREP_NOT_CONNECTED;
	goto fail;
    }
    if (no_suspend) {
	failure = ERTS_DSIG_PREP_CONNECTED;
	erts_smp_mtx_lock(&dep->qlock);
	if (dep->qflgs & ERTS_DE_QFLG_BUSY)
	    failure = ERTS_DSIG_PREP_WOULD_SUSPEND;
	erts_smp_mtx_unlock(&dep->qlock);
	if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND)
	    goto fail;
    }
    dsdp->proc = proc;
    dsdp->dep = dep;
    dsdp->cid = dep->cid;
    dsdp->connection_id = dep->connection_id;
    dsdp->no_suspend = no_suspend;
    if (dspl == ERTS_DSP_NO_LOCK)
	erts_smp_de_runlock(dep);
    return ERTS_DSIG_PREP_CONNECTED;

 fail:
    if (dspl == ERTS_DSP_RWLOCK)
	erts_smp_de_rwunlock(dep);
    else
	erts_smp_de_runlock(dep);
    return failure;

}

ERTS_GLB_INLINE
void erts_schedule_dist_command(Port *prt, DistEntry *dist_entry)
{
    DistEntry *dep;
    Eterm id;

    if (prt) {
	ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt));
	ASSERT((erts_atomic32_read_nob(&prt->state)
		& ERTS_PORT_SFLGS_DEAD) == 0);
	ASSERT(prt->dist_entry);

	dep = prt->dist_entry;
	id = prt->common.id;
    }
    else {
	ASSERT(dist_entry);
	ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&dist_entry->rwmtx)
			   || erts_lc_rwmtx_is_rwlocked(&dist_entry->rwmtx));
	ASSERT(is_internal_port(dist_entry->cid));

 	dep = dist_entry;
	id = dep->cid;
    }

    if (!erts_smp_atomic_xchg_mb(&dep->dist_cmd_scheduled, 1))
	erts_port_task_schedule(id, &dep->dist_cmd, ERTS_PORT_TASK_DIST_CMD);
}

#endif
# 218 "beam/dist.h"

typedef struct {
    ErtsLink *d_lnk;
    ErtsLink *d_sub_lnk;
} ErtsDistLinkData;

ERTS_GLB_INLINE void erts_remove_dist_link(ErtsDistLinkData *,
					   Eterm,
					   Eterm,
					   DistEntry *);
ERTS_GLB_INLINE int erts_was_dist_link_removed(ErtsDistLinkData *);
ERTS_GLB_INLINE void erts_destroy_dist_link(ErtsDistLinkData *);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE void
erts_remove_dist_link(ErtsDistLinkData *dldp,
		      Eterm lid,
		      Eterm rid,
		      DistEntry *dep)
{
    erts_smp_de_links_lock(dep);
    dldp->d_lnk = erts_lookup_link(dep->nlinks, lid);
    if (!dldp->d_lnk)
	dldp->d_sub_lnk = NULL;
    else {
	dldp->d_sub_lnk = erts_remove_link(&ERTS_LINK_ROOT(dldp->d_lnk), rid);
	dldp->d_lnk = (ERTS_LINK_ROOT(dldp->d_lnk)
		       ? NULL
		       : erts_remove_link(&dep->nlinks, lid));
    }
    erts_smp_de_links_unlock(dep);
}

ERTS_GLB_INLINE int
erts_was_dist_link_removed(ErtsDistLinkData *dldp)
{
    return dldp->d_sub_lnk != NULL;
}

ERTS_GLB_INLINE void
erts_destroy_dist_link(ErtsDistLinkData *dldp)
{
    if (dldp->d_lnk)
	erts_destroy_link(dldp->d_lnk);
    if (dldp->d_sub_lnk)
	erts_destroy_link(dldp->d_sub_lnk);	
}

#endif
# 268 "beam/dist.h"



/* Define for testing */
/* #define EXTREME_TTB_TRAPPING 1 */

#ifndef EXTREME_TTB_TRAPPING
#define TERM_TO_BINARY_LOOP_FACTOR 32
#else
# 277 "beam/dist.h"
#define TERM_TO_BINARY_LOOP_FACTOR 1
#endif
# 279 "beam/dist.h"

typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState;
typedef struct TTBSizeContext_ {
    Uint flags;
    int level;
    Uint result;
    Eterm obj;
    ErtsWStack wstack;
} TTBSizeContext;

typedef struct TTBEncodeContext_ {
    Uint flags;
    int level;
    byte* ep;
    Eterm obj;
    ErtsWStack wstack;
    Binary *result_bin;
} TTBEncodeContext;

typedef struct {
    Uint real_size;
    Uint dest_len;
    byte *dbytes;
    Binary *result_bin;
    Binary *destination_bin;
    z_stream stream;
} TTBCompressContext;

typedef struct {
    int alive;
    TTBState state;
    union {
	TTBSizeContext sc;
	TTBEncodeContext ec;
	TTBCompressContext cc;
    } s;
} TTBContext;

enum erts_dsig_send_phase {
    ERTS_DSIG_SEND_PHASE_INIT,
    ERTS_DSIG_SEND_PHASE_MSG_SIZE,
    ERTS_DSIG_SEND_PHASE_ALLOC,
    ERTS_DSIG_SEND_PHASE_MSG_ENCODE,
    ERTS_DSIG_SEND_PHASE_FIN
};

struct erts_dsig_send_context {
    enum erts_dsig_send_phase phase;
    Sint reds;

    Eterm ctl;
    Eterm msg;
    int force_busy;
    Uint32 pass_through_size;
    Uint data_size, dhdr_ext_size;
    ErtsAtomCacheMap *acmp;
    ErtsDistOutputBuf *obuf;
    Uint32 flags;
    Process *c_p;
    union {
	TTBSizeContext sc;
	TTBEncodeContext ec;
    }u;
};

typedef struct {
    int suspend;

    Eterm ctl_heap[6];
    ErtsDSigData dsd;
    DistEntry* dep_to_deref;
    struct erts_dsig_send_context dss;

    Eterm return_term;
}ErtsSendContext;


/*
 * erts_dsig_send_* return values.
 */
#define ERTS_DSIG_SEND_OK	0
#define ERTS_DSIG_SEND_YIELD	1
#define ERTS_DSIG_SEND_CONTINUE 2

extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm);
extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*);
extern int erts_dsig_send_exit_tt(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm);
extern int erts_dsig_send_unlink(ErtsDSigData *, Eterm, Eterm);
extern int erts_dsig_send_reg_msg(Eterm, Eterm, ErtsSendContext*);
extern int erts_dsig_send_group_leader(ErtsDSigData *, Eterm, Eterm);
extern int erts_dsig_send_exit(ErtsDSigData *, Eterm, Eterm, Eterm);
extern int erts_dsig_send_exit2(ErtsDSigData *, Eterm, Eterm, Eterm);
extern int erts_dsig_send_demonitor(ErtsDSigData *, Eterm, Eterm, Eterm, int);
extern int erts_dsig_send_monitor(ErtsDSigData *, Eterm, Eterm, Eterm);
extern int erts_dsig_send_m_exit(ErtsDSigData *, Eterm, Eterm, Eterm, Eterm);

extern int erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx);
extern void erts_dsend_context_dtor(Binary*);
extern Eterm erts_dsend_export_trap_context(Process* p, ErtsSendContext* ctx);

extern int erts_dist_command(Port *prt, int reds);
extern void erts_dist_port_not_busy(Port *prt);
extern void erts_kill_dist_connection(DistEntry *dep, Uint32);

extern Uint erts_dist_cache_size(void);


#endif
# 387 "beam/dist.h"
# 38 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "beam_bp.h"
#endif /* expanded by -frewrite-includes */
# 38 "beam/beam_emu.c"
# 1 "beam/beam_bp.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2000-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef _BEAM_BP_H
#define _BEAM_BP_H

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/beam_bp.h"
# 26 "beam/beam_bp.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_vm.h"
#endif /* expanded by -frewrite-includes */
# 26 "beam/beam_bp.h"
# 27 "beam/beam_bp.h"
#if 0 /* expanded by -frewrite-includes */
#include "global.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/beam_bp.h"
# 28 "beam/beam_bp.h"

typedef struct {
    Eterm pid;
    Sint  count;
    ErtsMonotonicTime time;
} bp_data_time_item_t;

typedef struct {
    Uint n;
    Uint used;
    bp_data_time_item_t *item;
} bp_time_hash_t;

typedef struct bp_data_time {     /* Call time */
    Uint n;
    bp_time_hash_t *hash;
    erts_refc_t refc;
} BpDataTime;

typedef struct {
    ErtsMonotonicTime time;
    BeamInstr *pc;
} process_breakpoint_time_t; /* used within psd */

typedef struct {
    erts_smp_atomic_t acount;
    erts_refc_t refc;
} BpCount;

typedef struct {
    erts_smp_atomic_t pid;
    erts_refc_t refc;
} BpMetaPid;

typedef struct generic_bp_data {
    Uint flags;
    Binary* local_ms;		/* Match spec for local call trace */
    Binary* meta_ms;		/* Match spec for meta trace */
    BpMetaPid* meta_pid;	/* Meta trace pid */
    BpCount* count;		/* For call count */
    BpDataTime* time;		/* For time trace */
} GenericBpData;

#define ERTS_NUM_BP_IX 2

typedef struct generic_bp {
    BeamInstr orig_instr;
    GenericBpData data[ERTS_NUM_BP_IX];
} GenericBp;

#define ERTS_BP_CALL_TIME_SCHEDULE_IN      (0)
#define ERTS_BP_CALL_TIME_SCHEDULE_OUT     (1)
#define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)

#ifdef ERTS_SMP
#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1)
#else
# 85 "beam/beam_bp.h"
#define bp_sched2ix_proc(p) (0)
#endif
# 87 "beam/beam_bp.h"

enum erts_break_op{
    erts_break_nop   =  0, /* Must be false */
    erts_break_set   = !0, /* Must be true */
    erts_break_reset,
    erts_break_stop
};

typedef Uint32 ErtsBpIndex;

typedef struct {
    BeamInstr* pc;
    Module* mod;
} BpFunction;

typedef struct {
    Uint matched;		/* Number matched */
    BpFunction* matching;	/* Matching functions */
} BpFunctions;

/*
** Function interface exported from beam_bp.c
*/

void erts_bp_init(void);

void erts_prepare_bp_staging(void);
void erts_commit_staged_bp(void);

ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void);
ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void);

void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified);
void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified);
void erts_bp_free_matched_functions(BpFunctions* f);

void erts_install_breakpoints(BpFunctions* f);
void erts_uninstall_breakpoints(BpFunctions* f);
void erts_consolidate_bp_data(BpFunctions* f, int local);
void erts_consolidate_bif_bp_data(void);

void erts_set_trace_break(BpFunctions *f, Binary *match_spec);
void erts_clear_trace_break(BpFunctions *f);

void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local);
void erts_clear_call_trace_bif(BeamInstr *pc, int local);

void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec,
			  Eterm tracer_pid);
void erts_clear_mtrace_break(BpFunctions *f);
void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec,
			 Eterm tracer_pid);
void erts_clear_mtrace_bif(BeamInstr *pc);

void erts_set_debug_break(BpFunctions *f);
void erts_clear_debug_break(BpFunctions *f);
void erts_set_count_break(BpFunctions *f, enum erts_break_op);
void erts_clear_count_break(BpFunctions *f);


void erts_clear_all_breaks(BpFunctions* f);
int erts_clear_module_break(Module *modp);
void erts_clear_export_break(Module *modp, BeamInstr* pc);

BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg);
BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args,
		      Uint32 *ret_flags, Eterm *tracer_pid);

int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local);
int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret,
			 Eterm *tracer_pid_rte);
int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret,
		       Eterm *tracer_pid_ret);
int erts_is_native_break(BeamInstr *pc);
int erts_is_count_break(BeamInstr *pc, Uint *count_ret);
int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time);

void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt);
void erts_trace_time_return(Process* c_p, BeamInstr* pc);
void erts_schedule_time_break(Process *p, Uint out);
void erts_set_time_break(BpFunctions *f, enum erts_break_op);
void erts_clear_time_break(BpFunctions *f);

int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time);
void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op);
void erts_clear_time_trace_bif(BeamInstr *pc);

BeamInstr *erts_find_local_func(Eterm mfa[3]);

ERTS_GLB_INLINE Uint erts_bp_sched2ix(void);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE Uint erts_bp_sched2ix(void)
{
#ifdef ERTS_SMP
    ErtsSchedulerData *esdp;
    esdp = erts_get_scheduler_data();
    return esdp->no - 1;
#else
# 186 "beam/beam_bp.h"
    return 0;
#endif
# 188 "beam/beam_bp.h"
}

extern erts_smp_atomic32_t erts_active_bp_index;
extern erts_smp_atomic32_t erts_staging_bp_index;

ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void)
{
    return erts_smp_atomic32_read_nob(&erts_active_bp_index);
}

ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void)
{
    return erts_smp_atomic32_read_nob(&erts_staging_bp_index);
}
#endif
# 203 "beam/beam_bp.h"

#endif /* _BEAM_BP_H */
# 205 "beam/beam_bp.h"
# 39 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "beam_catches.h"
#endif /* expanded by -frewrite-includes */
# 39 "beam/beam_emu.c"
# 1 "beam/beam_catches.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2000-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifndef __BEAM_CATCHES_H
#define __BEAM_CATCHES_H

#ifdef HAVE_CONFIG_H
#if 0 /* expanded by -frewrite-includes */
#  include "config.h"
#endif /* expanded by -frewrite-includes */
# 25 "beam/beam_catches.h"
# 1 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h" 1
/* armv6-portbld-freebsd11.0/config.h.  Generated by configure.  */
/* config.h.in.  Generated from configure.in by autoheader.  */


#define GHBN_R_SOLARIS  2
#define GHBN_R_AIX      3
#define GHBN_R_GLIBC    4


/* Assumed cache-line size (in bytes) */
#define ASSUMED_CACHE_LINE_SIZE 64

/* Define the brk() argument type. */
#define BRK_ARG_TYPE const void *

/* Define the brk() return type. */
#define BRK_RET_TYPE int

/* Modern style mcontext_t in MacOSX */
/* #undef DARWIN_MODERN_MCONTEXT */

/* Define if you need to include rpc/types.h to get INADDR_LOOPBACK defined */
/* #undef DEF_INADDR_LOOPBACK_IN_RPC_TYPES_H */

/* Define if you need to include winsock2.h to get INADDR_LOOPBACK defined */
/* #undef DEF_INADDR_LOOPBACK_IN_WINSOCK2_H */

/* Define if you want to disable child waiter thread */
/* #undef DISABLE_CHILD_WAITER_THREAD */

/* Define if you want to disable vfork. */
/* #undef DISABLE_VFORK */

/* Define to 1 if your processor stores the words in a double in middle-endian
   format (like some ARMs). */
/* #undef DOUBLE_MIDDLE_ENDIAN */

/* Define if you want to enable child waiter thread */
/* #undef ENABLE_CHILD_WAITER_THREAD */

/* Dirty scheduler support */
/* #undef ERL_DRV_DIRTY_SCHEDULER_SUPPORT */

/* Dirty scheduler support */
/* #undef ERL_NIF_DIRTY_SCHEDULER_SUPPORT */

/* Define if sbrk()/brk() wrappers can track malloc()s core memory use */
/* #undef ERTS_BRK_WRAPPERS_CAN_TRACK_MALLOC */

/* Define to override the default number of write_concurrency locks */
/* #undef ERTS_DB_HASH_LOCK_CNT */

/* Define if the emulator supports dirty schedulers */
/* #undef ERTS_DIRTY_SCHEDULERS */

/* The only reason ERTS_EMU_CMDLINE_FLAGS exists is to force modification of
   config.h when the emulator command line flags are modified by configure */
#define ERTS_EMU_CMDLINE_FLAGS " -O -pipe -mfloat-abi=softfp -fno-strict-aliasing -I/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0   -g  -I/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0    -DUSE_THREADS -D_THREAD_SAFE -D_REENTRANT -DPOSIX_THREADS  -Werror=return-type  -Wall -Wstrict-prototypes -Wmissing-prototypes -Wdeclaration-after-statement"

/* Define if you have kernel poll and want to use it */
#define ERTS_ENABLE_KERNEL_POLL 1

/* Define if OS monotonic clock is corrected */
/* #undef ERTS_HAVE_CORRECTED_OS_MONOTONIC_TIME */

/* Define if you have a low resolution OS monotonic clock */
/* #undef ERTS_HAVE_LOW_RESOLUTION_OS_MONOTONIC_LOW */

/* Define if the smp emulator is built */
#define ERTS_HAVE_SMP_EMU 1

/* Define if dlopen() needs to be called before first call to dlerror() */
/* #undef ERTS_NEED_DLOPEN_BEFORE_DLERROR */

/* Save compile time? */
#define ERTS_SAVED_COMPILE_TIME 1

/* Define if poll() should be used instead of select() */
#define ERTS_USE_POLL 1

/* Define if __after_morecore_hook can track malloc()s core memory use. */
/* #undef ERTS___AFTER_MORECORE_HOOK_CAN_TRACK_MALLOC */

/* Define if bigendian */
/* #undef ETHR_BIGENDIAN */

/* Define if gcc wont let you clobber ebx with cmpxchg8b and position
   independent code */
/* #undef ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX */

/* Define if you get a register shortage with cmpxchg8b and position
   independent code */
/* #undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE */

/* Define if you want to disable native ethread implementations */
/* #undef ETHR_DISABLE_NATIVE_IMPLS */

/* Define if you want to force usage of pthread rwlocks */
/* #undef ETHR_FORCE_PTHREAD_RWLOCK */

/* Define if you use a gcc that supports the double word cmpxchg instruction
   */
/* #undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT */

/* Define if you use a gcc that supports -msse2 and understand sse2 specific
   asm statements */
/* #undef ETHR_GCC_HAVE_SSE2_ASM_SUPPORT */

/* Define if you have a clock_gettime() with a monotonic clock */
#define ETHR_HAVE_CLOCK_GETTIME_MONOTONIC 1

/* Define if you have all ethread defines */
#define ETHR_HAVE_ETHREAD_DEFINES 1

/* Define as a boolean indicating whether you have a gcc compatible compiler
   capable of generating the ARM DMB instruction, and are compiling for an ARM
   processor with ARM DMB instruction support, or not */
#define ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION 0

/* Define as a boolean indicating whether you have a gcc __atomic builtins or
   not */
#define ETHR_HAVE_GCC___ATOMIC_BUILTINS 1

/* Define if you have a monotonic gethrtime() */
/* #undef ETHR_HAVE_GETHRTIME */

/* Define if you have libatomic_ops atomic operations */
/* #undef ETHR_HAVE_LIBATOMIC_OPS */

/* Define if you have a linux futex implementation. */
/* #undef ETHR_HAVE_LINUX_FUTEX */

/* Define if you have a mach clock_get_time() with a monotonic clock */
/* #undef ETHR_HAVE_MACH_CLOCK_GET_TIME */

/* Define if the pthread.h header file is in pthread/mit directory. */
/* #undef ETHR_HAVE_MIT_PTHREAD_H */

/* Define if you have the pthread_attr_setguardsize function. */
#define ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE 1

/* Define if pthread_cond_timedwait() can be used with a monotonic clock */
#define ETHR_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC 1

/* Define if you have ibm style pthread_getname_np */
/* #undef ETHR_HAVE_PTHREAD_GETNAME_NP_2 */

/* Define if you have linux style pthread_getname_np */
/* #undef ETHR_HAVE_PTHREAD_GETNAME_NP_3 */

/* Define if you have the <pthread.h> header file. */
#define ETHR_HAVE_PTHREAD_H 1

/* Define if you have the pthread_rwlockattr_setkind_np() function. */
/* #undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP */

/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock
   attribute. */
/* #undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP */

/* Define if you have darwin style pthread_setname_np */
/* #undef ETHR_HAVE_PTHREAD_SETNAME_NP_1 */

/* Define if you have linux style pthread_setname_np */
/* #undef ETHR_HAVE_PTHREAD_SETNAME_NP_2 */

/* Define if you have bsd style pthread_set_name_np */
/* #undef ETHR_HAVE_PTHREAD_SET_NAME_NP_2 */

/* Define if you have the pthread_spin_lock function. */
#define ETHR_HAVE_PTHREAD_SPIN_LOCK 1

/* Define if you have the pthread_yield() function. */
#define ETHR_HAVE_PTHREAD_YIELD 1

/* Define if you have the <sched.h> header file. */
#define ETHR_HAVE_SCHED_H 1

/* Define if you have the sched_yield() function. */
#define ETHR_HAVE_SCHED_YIELD 1

/* Define if you have the <sys/time.h> header file. */
#define ETHR_HAVE_SYS_TIME_H 1

/* Define if you have _InterlockedAnd() */
/* #undef ETHR_HAVE__INTERLOCKEDAND */

/* Define if you have _InterlockedAnd64() */
/* #undef ETHR_HAVE__INTERLOCKEDAND64 */

/* Define if you have _InterlockedCompareExchange() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE */

/* Define if you have _InterlockedCompareExchange128() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128 */

/* Define if you have _InterlockedCompareExchange64() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 */

/* Define if you have _InterlockedCompareExchange64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ */

/* Define if you have _InterlockedCompareExchange64_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL */

/* Define if you have _InterlockedCompareExchange_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ */

/* Define if you have _InterlockedCompareExchange_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL */

/* Define if you have _InterlockedDecrement() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT */

/* Define if you have _InterlockedDecrement64() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT64 */

/* Define if you have _InterlockedDecrement64_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL */

/* Define if you have _InterlockedDecrement_rel() */
/* #undef ETHR_HAVE__INTERLOCKEDDECREMENT_REL */

/* Define if you have _InterlockedExchange() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGE */

/* Define if you have _InterlockedExchange64() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 */

/* Define if you have _InterlockedExchangeAdd() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD */

/* Define if you have _InterlockedExchangeAdd64() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 */

/* Define if you have _InterlockedExchangeAdd64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ */

/* Define if you have _InterlockedExchangeAdd_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ */

/* Define if you have _InterlockedIncrement() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT */

/* Define if you have _InterlockedIncrement64() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT64 */

/* Define if you have _InterlockedIncrement64_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ */

/* Define if you have _InterlockedIncrement_acq() */
/* #undef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ */

/* Define if you have _InterlockedOr() */
/* #undef ETHR_HAVE__INTERLOCKEDOR */

/* Define if you have _InterlockedOr64() */
/* #undef ETHR_HAVE__INTERLOCKEDOR64 */

/* Define as a bitmask corresponding to the word sizes that
   __atomic_add_fetch() can handle on your system */
#define ETHR_HAVE___atomic_add_fetch 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_compare_exchange_n() can handle on your system */
#define ETHR_HAVE___atomic_compare_exchange_n 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_fetch_and() can handle on your system */
#define ETHR_HAVE___atomic_fetch_and 12

/* Define as a bitmask corresponding to the word sizes that
   __atomic_fetch_or() can handle on your system */
#define ETHR_HAVE___atomic_fetch_or 12

/* Define as a bitmask corresponding to the word sizes that __atomic_load_n()
   can handle on your system */
#define ETHR_HAVE___atomic_load_n 12

/* Define as a bitmask corresponding to the word sizes that __atomic_store_n()
   can handle on your system */
#define ETHR_HAVE___atomic_store_n 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_add_and_fetch() can handle on your system */
#define ETHR_HAVE___sync_add_and_fetch 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_fetch_and_and() can handle on your system */
#define ETHR_HAVE___sync_fetch_and_and 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_fetch_and_or() can handle on your system */
#define ETHR_HAVE___sync_fetch_and_or 12

/* Define as a bitmask corresponding to the word sizes that
   __sync_synchronize() can handle on your system */
#define ETHR_HAVE___sync_synchronize ~0

/* Define as a bitmask corresponding to the word sizes that
   __sync_val_compare_and_swap() can handle on your system */
#define ETHR_HAVE___sync_val_compare_and_swap 12

/* Define if you want to modify the default stack size */
/* #undef ETHR_MODIFIED_DEFAULT_STACK_SIZE */

/* Define to the monotonic clock id to use */
#define ETHR_MONOTONIC_CLOCK_ID CLOCK_MONOTONIC

/* Define if you need the <nptl/pthread.h> header file. */
/* #undef ETHR_NEED_NPTL_PTHREAD_H */

/* Define if you have OSE style threads */
/* #undef ETHR_OSE_THREADS */

/* Define if you prefer gcc native ethread implementations */
/* #undef ETHR_PREFER_GCC_NATIVE_IMPLS */

/* Define if you prefer libatomic_ops native ethread implementations */
/* #undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS */

/* Define if you have pthreads */
#define ETHR_PTHREADS 1

/* Define if pthread_yield() returns an int. */
/* #undef ETHR_PTHREAD_YIELD_RET_INT */

/* Define if sched_yield() returns an int. */
#define ETHR_SCHED_YIELD_RET_INT 1

/* Define to the size of AO_t if libatomic_ops is used */
/* #undef ETHR_SIZEOF_AO_T */

/* Define to the size of int */
#define ETHR_SIZEOF_INT 4

/* Define to the size of long */
#define ETHR_SIZEOF_LONG 4

/* Define to the size of long long */
#define ETHR_SIZEOF_LONG_LONG 8

/* Define to the size of pointers */
#define ETHR_SIZEOF_PTR 4

/* Define to the size of __int128_t */
#define ETHR_SIZEOF___INT128_T 0

/* Define to the size of __int64 */
#define ETHR_SIZEOF___INT64 0

/* Define if you want to enable check for native ethread implementations */
#define ETHR_SMP_REQUIRE_NATIVE_IMPLS 1

/* Define if only run in Sparc PSO, or TSO mode */
/* #undef ETHR_SPARC_PSO */

/* Define if run in Sparc RMO, PSO, or TSO mode */
/* #undef ETHR_SPARC_RMO */

/* Define if only run in Sparc TSO mode */
/* #undef ETHR_SPARC_TSO */

/* Define if you can safely include both <sys/time.h> and <time.h>. */
#define ETHR_TIME_WITH_SYS_TIME 1

/* Define as a boolean indicating whether you trust gcc's __atomic_* builtins
   memory barrier implementations, or not */
#define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS 0

/* Define if you have win32 threads */
/* #undef ETHR_WIN32_THREADS */

/* Define if x86/x86_64 out of order instructions should be synchronized */
/* #undef ETHR_X86_OUT_OF_ORDER */

/* Define if building a halfword-heap 64bit emulator */
/* #undef HALFWORD_HEAP_EMULATOR */

/* Define to 1 if you have the <arpa/nameser.h> header file. */
#define HAVE_ARPA_NAMESER_H 1

/* Define to 1 if you have the `brk' function. */
#define HAVE_BRK 1

/* Define to 1 if you have the `clock_getres' function. */
#define HAVE_CLOCK_GETRES 1

/* define if clock_gettime() works for getting process time */
/* #undef HAVE_CLOCK_GETTIME_CPU_TIME */

/* Define if you have clock_gettime(CLOCK_MONOTONIC_RAW, _) */
/* #undef HAVE_CLOCK_GETTIME_MONOTONIC_RAW */

/* Define to 1 if you have the `clock_get_attributes' function. */
/* #undef HAVE_CLOCK_GET_ATTRIBUTES */

/* Define to 1 if you have the `closefrom' function. */
#define HAVE_CLOSEFROM 1

/* Define if you have a decl of fread that conflicts with int fread */
#define HAVE_CONFLICTING_FREAD_DECLARATION 1

/* Define if you have a putenv() that stores a copy of the key-value pair */
/* #undef HAVE_COPYING_PUTENV */

/* Define if you have cpuset_getaffinity/cpuset_setaffinity */
#define HAVE_CPUSET_xETAFFINITY 1

/* Define to 1 if you have the declaration of `getrlimit', and to 0 if you
   don't. */
#define HAVE_DECL_GETRLIMIT 1

/* Define to 1 if you have the declaration of `IN6ADDR_ANY_INIT', and to 0 if
   you don't. */
#define HAVE_DECL_IN6ADDR_ANY_INIT 1

/* Define to 1 if you have the declaration of `IN6ADDR_LOOPBACK_INIT', and to
   0 if you don't. */
#define HAVE_DECL_IN6ADDR_LOOPBACK_INIT 1

/* Define to 1 if you have the declaration of `IPV6_V6ONLY', and to 0 if you
   don't. */
#define HAVE_DECL_IPV6_V6ONLY 1

/* Define to 1 if you have the declaration of `posix2time', and to 0 if you
   don't. */
#define HAVE_DECL_POSIX2TIME 0

/* Define to 1 if you have the declaration of `RLIMIT_STACK', and to 0 if you
   don't. */
#define HAVE_DECL_RLIMIT_STACK 1

/* Define to 1 if you have the declaration of `SCTPS_BOUND', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_BOUND 0

/* Define to 1 if you have the declaration of `SCTPS_COOKIE_ECHOED', and to 0
   if you don't. */
#define HAVE_DECL_SCTPS_COOKIE_ECHOED 0

/* Define to 1 if you have the declaration of `SCTPS_COOKIE_WAIT', and to 0 if
   you don't. */
#define HAVE_DECL_SCTPS_COOKIE_WAIT 0

/* Define to 1 if you have the declaration of `SCTPS_ESTABLISHED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTPS_ESTABLISHED 0

/* Define to 1 if you have the declaration of `SCTPS_IDLE', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_IDLE 0

/* Define to 1 if you have the declaration of `SCTPS_LISTEN', and to 0 if you
   don't. */
#define HAVE_DECL_SCTPS_LISTEN 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_ACK_SENT', and
   to 0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_ACK_SENT 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_PENDING', and to
   0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_PENDING 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_RECEIVED', and
   to 0 if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_RECEIVED 0

/* Define to 1 if you have the declaration of `SCTPS_SHUTDOWN_SENT', and to 0
   if you don't. */
#define HAVE_DECL_SCTPS_SHUTDOWN_SENT 0

/* Define to 1 if you have the declaration of `SCTP_ABORT', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_ABORT 1

/* Define to 1 if you have the declaration of `SCTP_ADDR_CONFIRMED', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_ADDR_CONFIRMED 1

/* Define to 1 if you have the declaration of `SCTP_ADDR_OVER', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_ADDR_OVER 1

/* Define to 1 if you have the declaration of `SCTP_BOUND', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_BOUND 1

/* Define to 1 if you have the declaration of `SCTP_CLOSED', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_CLOSED 1

/* Define to 1 if you have the declaration of `SCTP_COOKIE_ECHOED', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_COOKIE_ECHOED 1

/* Define to 1 if you have the declaration of `SCTP_COOKIE_WAIT', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_COOKIE_WAIT 1

/* Define to 1 if you have the declaration of `SCTP_DELAYED_ACK_TIME', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_DELAYED_ACK_TIME 0

/* Define to 1 if you have the declaration of `SCTP_EMPTY', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_EMPTY 0

/* Define to 1 if you have the declaration of `SCTP_EOF', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_EOF 1

/* Define to 1 if you have the declaration of `SCTP_ESTABLISHED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_ESTABLISHED 1

/* Define to 1 if you have the declaration of `SCTP_LISTEN', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_LISTEN 1

/* Define to 1 if you have the declaration of `SCTP_SENDALL', and to 0 if you
   don't. */
#define HAVE_DECL_SCTP_SENDALL 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_ACK_SENT', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_ACK_SENT 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_PENDING', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_PENDING 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_RECEIVED', and to
   0 if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_RECEIVED 1

/* Define to 1 if you have the declaration of `SCTP_SHUTDOWN_SENT', and to 0
   if you don't. */
#define HAVE_DECL_SCTP_SHUTDOWN_SENT 1

/* Define to 1 if you have the declaration of `SCTP_UNCONFIRMED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_UNCONFIRMED 1

/* Define to 1 if you have the declaration of `SCTP_UNORDERED', and to 0 if
   you don't. */
#define HAVE_DECL_SCTP_UNORDERED 1

/* Define to 1 if you have the declaration of `setrlimit', and to 0 if you
   don't. */
#define HAVE_DECL_SETRLIMIT 1

/* Define to 1 if you have the declaration of `time2posix', and to 0 if you
   don't. */
#define HAVE_DECL_TIME2POSIX 0

/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
   */
#define HAVE_DIRENT_H 1

/* Define to 1 if you have the <dlfcn.h> header file. */
#define HAVE_DLFCN_H 1

/* Define to 1 if you have the `dlopen' function. */
#define HAVE_DLOPEN 1

/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
/* #undef HAVE_DOPRNT */

/* Define if you have the 'end' symbol */
#define HAVE_END_SYMBOL 1

/* Define if you have a working fallocate() */
/* #undef HAVE_FALLOCATE */

/* Define to 1 if you have the <fcntl.h> header file. */
#define HAVE_FCNTL_H 1

/* Define to 1 if you have the `fdatasync' function. */
/* #undef HAVE_FDATASYNC */

/* Define to 1 if you have the `finite' function. */
#define HAVE_FINITE 1

/* Define to 1 if you have the `flockfile' function. */
#define HAVE_FLOCKFILE 1

/* Define to 1 if you have the `fork' function. */
#define HAVE_FORK 1

/* Define to 1 if you have the `fpsetmask' function. */
#define HAVE_FPSETMASK 1

/* Define to 1 if you have the `fstat' function. */
#define HAVE_FSTAT 1

/* Define if you have fwrite_unlocked */
/* #undef HAVE_FWRITE_UNLOCKED */

/* Define to 1 if you have a good `getaddrinfo' function. */
#define HAVE_GETADDRINFO 1

/* Define to 1 if you have the `gethostbyname2' function. */
#define HAVE_GETHOSTBYNAME2 1

/* Define to flavour of gethostbyname_r */
/* #undef HAVE_GETHOSTBYNAME_R */

/* Define to 1 if you have the `gethrtime' function. */
/* #undef HAVE_GETHRTIME */

/* define if gethrvtime() works and uses ioctl() to /proc/self */
/* #undef HAVE_GETHRVTIME_PROCFS_IOCTL */

/* Define to 1 if you have the `getifaddrs' function. */
#define HAVE_GETIFADDRS 1

/* Define to 1 if you have the `getipnodebyaddr' function. */
#define HAVE_GETIPNODEBYADDR 1

/* Define to 1 if you have the `getipnodebyname' function. */
#define HAVE_GETIPNODEBYNAME 1

/* Define to 1 if you have a good `getnameinfo' function. */
#define HAVE_GETNAMEINFO 1

/* Define to 1 if you have the `gettimeofday' function. */
#define HAVE_GETTIMEOFDAY 1

/* Define to 1 if you have the `gmtime_r' function. */
#define HAVE_GMTIME_R 1

/* Define to 1 if you have the <ieeefp.h> header file. */
#define HAVE_IEEEFP_H 1

/* Define to 1 if you have the `ieee_handler' function. */
/* #undef HAVE_IEEE_HANDLER */

/* Define to 1 if you have the <ifaddrs.h> header file. */
#define HAVE_IFADDRS_H 1

/* Define if ipv6 is present */
#define HAVE_IN6 1

/* Define to 1 if you have the variable in6addr_any declared. */
#define HAVE_IN6ADDR_ANY 1

/* Define to 1 if you have the variable in6addr_loopback declared. */
#define HAVE_IN6ADDR_LOOPBACK 1

/* Define to 1 if you have the `inet_pton' function. */
#define HAVE_INET_PTON 1

/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1

/* Early linux used in_addr6 instead of in6_addr, define if you have this */
/* #undef HAVE_IN_ADDR6_STRUCT */

/* Define to 1 if you have the `isfinite' function. */
/* #undef HAVE_ISFINITE */

/* Define to 1 if you have the `isinf' function. */
#define HAVE_ISINF 1

/* Define to 1 if you have the `isnan' function. */
#define HAVE_ISNAN 1

/* Define if you have kstat */
/* #undef HAVE_KSTAT */

/* Define to 1 if you have the <langinfo.h> header file. */
#define HAVE_LANGINFO_H 1

/* Define to 1 if you have the `dl' library (-ldl). */
/* #undef HAVE_LIBDL */

/* Define to 1 if you have the `dlpi' library (-ldlpi). */
/* #undef HAVE_LIBDLPI */

/* Define to 1 if you have the <libdlpi.h> header file. */
/* #undef HAVE_LIBDLPI_H */

/* Define to 1 if you have the `inet' library (-linet). */
/* #undef HAVE_LIBINET */

/* Define to 1 if you have the `m' library (-lm). */
#define HAVE_LIBM 1

/* Define to 1 if you have the `util' library (-lutil). */
#define HAVE_LIBUTIL 1

/* Define to 1 if you have the `z' library (-lz). */
#define HAVE_LIBZ 1

/* Define to 1 if you have the <limits.h> header file. */
#define HAVE_LIMITS_H 1

/* Define to 1 if you have the <linux/falloc.h> header file. */
/* #undef HAVE_LINUX_FALLOC_H */

/* Define to 1 if you have the `localtime_r' function. */
#define HAVE_LOCALTIME_R 1

/* Define to 1 if you have the `log2' function. */
#define HAVE_LOG2 1

/* Define to 1 if you have the <malloc.h> header file. */
/* #undef HAVE_MALLOC_H */

/* Define to 1 if you have the `mallopt' function. */
/* #undef HAVE_MALLOPT */

/* Define to 1 if you have the `memcpy' function. */
#define HAVE_MEMCPY 1

/* Define to 1 if you have the `memmove' function. */
#define HAVE_MEMMOVE 1

/* Define to 1 if you have the <memory.h> header file. */
#define HAVE_MEMORY_H 1

/* Define if the pthread.h header file is in pthread/mit directory. */
/* #undef HAVE_MIT_PTHREAD_H */

/* Define to 1 if you have the `mlockall' function. */
#define HAVE_MLOCKALL 1

/* Define to 1 if you have the `mmap' function. */
#define HAVE_MMAP 1

/* Define if you have a monotonic erts_os_hrtime() implementation */
/* #undef HAVE_MONOTONIC_ERTS_SYS_HRTIME */

/* Define to 1 if you have the `mremap' function. */
/* #undef HAVE_MREMAP */

/* Define if setsockopt() accepts multicast options */
#define HAVE_MULTICAST_SUPPORT 1

/* Define to 1 if you have the <ndir.h> header file, and it defines `DIR'. */
/* #undef HAVE_NDIR_H */

/* Define to 1 if you have the <netpacket/packet.h> header file. */
/* #undef HAVE_NETPACKET_PACKET_H */

/* Define to 1 if you have the <net/errno.h> header file. */
/* #undef HAVE_NET_ERRNO_H */

/* Define to 1 if you have the <net/if_dl.h> header file. */
#define HAVE_NET_IF_DL_H 1

/* Define to 1 if you have the `nl_langinfo' function. */
#define HAVE_NL_LANGINFO 1

/* Define if you don't have a definition of INADDR_LOOPBACK */
/* #undef HAVE_NO_INADDR_LOOPBACK */

/* Define to 1 if you have the `openpty' function. */
#define HAVE_OPENPTY 1

/* Define if you have the "ose_spi/ose_spi.h" header file. */
/* #undef HAVE_OSE_SPI_H */

/* Define to 1 if you have the `poll' function. */
#define HAVE_POLL 1

/* Define to 1 if you have the <poll.h> header file. */
#define HAVE_POLL_H 1

/* Define to 1 if you have the `posix2time' function. */
#define HAVE_POSIX2TIME 1

/* Define to 1 if you have the `posix_fadvise' function. */
#define HAVE_POSIX_FADVISE 1

/* Define if you have a working posix_fallocate() */
#define HAVE_POSIX_FALLOCATE 

/* Define to 1 if you have the `posix_memalign' function. */
#define HAVE_POSIX_MEMALIGN 1

/* Define to 1 if you have the `ppoll' function. */
#define HAVE_PPOLL 1

/* Define to 1 if you have the `pread' function. */
#define HAVE_PREAD 1

/* Define if you have processor_bind functionality */
/* #undef HAVE_PROCESSOR_BIND */

/* Define if you have pset functionality */
/* #undef HAVE_PSET */

/* Define if you have the <pthread.h> header file. */
#define HAVE_PTHREAD_H 1

/* Define to 1 if you have the <pty.h> header file. */
/* #undef HAVE_PTY_H */

/* Define if you have putc_unlocked */
#define HAVE_PUTC_UNLOCKED 1

/* Define to 1 if you have the `pwrite' function. */
#define HAVE_PWRITE 1

/* Define to 1 if you have the `res_gethostbyname' function. */
/* #undef HAVE_RES_GETHOSTBYNAME */

/* Define to 1 if you have the `sbrk' function. */
#define HAVE_SBRK 1

/* Define to 1 if you have the <sched.h> header file. */
#define HAVE_SCHED_H 1

/* Define if you have sched_getaffinity/sched_setaffinity */
/* #undef HAVE_SCHED_xETAFFINITY */

/* Define to 1 if you have the `sctp_bindx' function. */
#define HAVE_SCTP_BINDX 1

/* Define to 1 if you have the `sctp_freeladdrs' function. */
#define HAVE_SCTP_FREELADDRS 1

/* Define to 1 if you have the `sctp_freepaddrs' function. */
#define HAVE_SCTP_FREEPADDRS 1

/* Define to 1 if you have the `sctp_getladdrs' function. */
#define HAVE_SCTP_GETLADDRS 1

/* Define to 1 if you have the `sctp_getpaddrs' function. */
#define HAVE_SCTP_GETPADDRS 1

/* Define to 1 if you have the <netinet/sctp.h> header file */
#define HAVE_SCTP_H 1

/* Define to 1 if you have the `sctp_peeloff' function. */
#define HAVE_SCTP_PEELOFF 1

/* Define to 1 if you have the <sdkddkver.h> header file. */
/* #undef HAVE_SDKDDKVER_H */

/* Define to 1 if you have the `sendfile' function. */
#define HAVE_SENDFILE 1

/* Define to 1 if you have the `sendfilev' function. */
/* #undef HAVE_SENDFILEV */

/* Define to 1 if you have the `setlocale' function. */
#define HAVE_SETLOCALE 1

/* Define to 1 if you have the `setns' function. */
/* #undef HAVE_SETNS */

/* Define to 1 if you have the <setns.h> header file. */
/* #undef HAVE_SETNS_H */

/* Define to 1 if you have the `setsid' function. */
#define HAVE_SETSID 1

/* Define if we have socklen_t */
#define HAVE_SOCKLEN_T 1

/* define if you have the Solaris/ultrasparc /dev/perfmon interface */
/* #undef HAVE_SOLARIS_SPARC_PERFMON */

/* Define if you have SO_BSDCOMPAT flag on sockets */
/* #undef HAVE_SO_BSDCOMPAT */

/* Define to 1 if you have the <stdint.h> header file. */
#define HAVE_STDINT_H 1

/* Define to 1 if you have the <stdlib.h> header file. */
#define HAVE_STDLIB_H 1

/* Define to 1 if you have the `strerror' function. */
#define HAVE_STRERROR 1

/* Define to 1 if you have the `strerror_r' function. */
#define HAVE_STRERROR_R 1

/* Define to 1 if you have the <strings.h> header file. */
#define HAVE_STRINGS_H 1

/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1

/* Define to 1 if you have the `strlcat' function. */
#define HAVE_STRLCAT 1

/* Define to 1 if you have the `strlcpy' function. */
#define HAVE_STRLCPY 1

/* Define to 1 if you have the `strncasecmp' function. */
#define HAVE_STRNCASECMP 1

/* Define to 1 if `ifr_enaddr' is member of `struct ifreq'. */
/* #undef HAVE_STRUCT_IFREQ_IFR_ENADDR */

/* Define to 1 if `ifr_hwaddr' is member of `struct ifreq'. */
/* #undef HAVE_STRUCT_IFREQ_IFR_HWADDR */

/* Define to 1 if `spp_flags' is member of `struct sctp_paddrparams'. */
#define HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS 1

/* Define to 1 if `spp_pathmtu' is member of `struct sctp_paddrparams'. */
#define HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_PATHMTU 1

/* Define to 1 if `spp_sackdelay' is member of `struct sctp_paddrparams'. */
/* #undef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY */

/* Define to 1 if `sre_data' is member of `struct sctp_remote_error'. */
#define HAVE_STRUCT_SCTP_REMOTE_ERROR_SRE_DATA 1

/* Define to 1 if `ssf_data' is member of `struct sctp_send_failed'. */
#define HAVE_STRUCT_SCTP_SEND_FAILED_SSF_DATA 1

/* Define to 1 if you have the <syslog.h> header file. */
#define HAVE_SYSLOG_H 1

/* Define if you have systemd daemon */
/* #undef HAVE_SYSTEMD_DAEMON */

/* Define to 1 if you have the <systemd/sd-daemon.h> header file. */
/* #undef HAVE_SYSTEMD_SD_DAEMON_H */

/* Define if you have <sys/devpoll.h> header file. */
/* #undef HAVE_SYS_DEVPOLL_H */

/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR'.
   */
/* #undef HAVE_SYS_DIR_H */

/* Define if you have the <sys/epoll.h> header file. */
/* #undef HAVE_SYS_EPOLL_H */

/* Define if you have <sys/event.h> header file. */
#define HAVE_SYS_EVENT_H 1

/* Define to 1 if you have the <sys/ioctl.h> header file. */
#define HAVE_SYS_IOCTL_H 1

/* Define to 1 if you have the <sys/ndir.h> header file, and it defines `DIR'.
   */
/* #undef HAVE_SYS_NDIR_H */

/* Define to 1 if you have the <sys/resource.h> header file */
#define HAVE_SYS_RESOURCE_H 1

/* Define to 1 if you have the <sys/sdt.h> header file. */
/* #undef HAVE_SYS_SDT_H */

/* Define to 1 if you have the <sys/socketio.h> header file. */
/* #undef HAVE_SYS_SOCKETIO_H */

/* Define to 1 if you have the <sys/socket.h> header file. */
#define HAVE_SYS_SOCKET_H 1

/* Define to 1 if you have the <sys/sockio.h> header file. */
#define HAVE_SYS_SOCKIO_H 1

/* Define to 1 if you have the <sys/stat.h> header file. */
#define HAVE_SYS_STAT_H 1

/* Define to 1 if you have the <sys/stropts.h> header file. */
/* #undef HAVE_SYS_STROPTS_H */

/* Define to 1 if you have the <sys/sysctl.h> header file. */
#define HAVE_SYS_SYSCTL_H 1

/* Define to 1 if you have the <sys/timerfd.h> header file. */
/* #undef HAVE_SYS_TIMERFD_H */

/* Define to 1 if you have the <sys/time.h> header file. */
#define HAVE_SYS_TIME_H 1

/* Define to 1 if you have the <sys/types.h> header file. */
#define HAVE_SYS_TYPES_H 1

/* Define to 1 if you have the <sys/uio.h> header file. */
#define HAVE_SYS_UIO_H 1

/* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */
#define HAVE_SYS_WAIT_H 1

/* Define if termcap functions exists */
#define HAVE_TERMCAP 1

/* Define to 1 if you have the `time2posix' function. */
#define HAVE_TIME2POSIX 1

/* Define to 1 if you have the <unistd.h> header file. */
#define HAVE_UNISTD_H 1

/* Define to 1 if you have the <util.h> header file. */
/* #undef HAVE_UTIL_H */

/* Define to 1 if you have the <utmp.h> header file. */
/* #undef HAVE_UTMP_H */

/* Define to 1 if you have the `vfork' function. */
#define HAVE_VFORK 1

/* Define to 1 if you have the <vfork.h> header file. */
/* #undef HAVE_VFORK_H */

/* Define to 1 if you have the `vprintf' function. */
#define HAVE_VPRINTF 1

/* Define to 1 if you have a `wcwidth' function. */
#define HAVE_WCWIDTH 1

/* Define to 1 if you have the <windows.h> header file. */
/* #undef HAVE_WINDOWS_H */

/* Define to 1 if you have the <winsock2.h> header file. */
/* #undef HAVE_WINSOCK2_H */

/* Define to 1 if `fork' works. */
#define HAVE_WORKING_FORK 1

/* Define if you have a working posix_openpt implementation */
#define HAVE_WORKING_POSIX_OPENPT 1

/* Define to 1 if `vfork' works. */
#define HAVE_WORKING_VFORK 1

/* Define to 1 if you have the `writev' function. */
#define HAVE_WRITEV 1

/* Define to 1 if you have the <ws2tcpip.h> header file. */
/* #undef HAVE_WS2TCPIP_H */

/* Define to 1 if you have the `_brk' function. */
#define HAVE__BRK 1

/* Define if you have the '_end' symbol */
#define HAVE__END_SYMBOL 1

/* Define to 1 if you have the `_sbrk' function. */
#define HAVE__SBRK 1

/* Define to 1 if you have the `__brk' function. */
/* #undef HAVE___BRK */

/* Define to 1 if you have the `__sbrk' function. */
/* #undef HAVE___SBRK */

/* Define to enable HiPE */
#define HIPE 1

/* Define to monotonic clock id to use */
#define HRTIME_CLOCK_ID CLOCK_UPTIME_PRECISE

/* Define as a string of monotonic clock id to use */
#define HRTIME_CLOCK_ID_STR "CLOCK_UPTIME_PRECISE"

/* define if h_errno is declared (in some way) in a system header file */
#define H_ERRNO_DECLARED 1

/* Define to monotonic clock id to use */
#define MONOTONIC_CLOCK_ID CLOCK_UPTIME

/* Define as a string of monotonic clock id to use */
#define MONOTONIC_CLOCK_ID_STR "CLOCK_UPTIME"

/* Define if netdb.h needs struct sockaddr_in ans in.h CAN be included before
   */
/* #undef NETDB_H_NEEDS_IN_H */

/* Define if floating points exceptions are non-existing/not reliable */
#define NO_FPE_SIGNALS 

/* Defined if no found C compiler can handle jump tables */
/* #undef NO_JUMP_TABLE */

/* Define if you dont have salen */
/* #undef NO_SA_LEN */

/* Define if you want to implement erts_os_monotonic_time() using
   clock_gettime() */
#define OS_MONOTONIC_TIME_USING_CLOCK_GETTIME 1

/* Define if you want to implement erts_os_monotonic_time() using gethrtime()
   */
/* #undef OS_MONOTONIC_TIME_USING_GETHRTIME */

/* Define if you want to implement erts_os_monotonic_time() using mach
   clock_get_time() */
/* #undef OS_MONOTONIC_TIME_USING_MACH_CLOCK_GET_TIME */

/* Define if you want to implement erts_os_monotonic_time() using times() */
/* #undef OS_MONOTONIC_TIME_USING_TIMES */

/* Define if you want to implement erts_os_system_time() using gettimeofday()
   */
/* #undef OS_SYSTEM_TIME_GETTIMEOFDAY */

/* Define if you want to implement erts_os_system_time() using clock_gettime()
   */
#define OS_SYSTEM_TIME_USING_CLOCK_GETTIME 1

/* Define if you want to implement erts_os_system_time() using mach
   clock_get_time() */
/* #undef OS_SYSTEM_TIME_USING_MACH_CLOCK_GET_TIME */

/* Define to the address where bug reports for this package should be sent. */
#define PACKAGE_BUGREPORT ""

/* Define to the full name of this package. */
#define PACKAGE_NAME ""

/* Define to the full name and version of this package. */
#define PACKAGE_STRING ""

/* Define to the one symbol short name of this package. */
#define PACKAGE_TARNAME ""

/* Define to the version of this package. */
#define PACKAGE_VERSION ""

/* Define if you wish to redefine FD_SETSIZE to be able to select on more fd
   */
/* #undef REDEFINE_FD_SETSIZE */

/* Define as the return type of signal handlers (`int' or `void'). */
#define RETSIGTYPE void

/* Define the sbrk() argument type. */
#define SBRK_ARG_TYPE intptr_t

/* Define the sbrk() return type. */
#define SBRK_RET_TYPE void *

/* The size of a `AO_t', as computed by sizeof. */
/* #undef SIZEOF_AO_T */

/* The size of a `char', as computed by sizeof. */
#define SIZEOF_CHAR 1

/* The size of a `int', as computed by sizeof. */
#define SIZEOF_INT 4

/* The size of a `long', as computed by sizeof. */
#define SIZEOF_LONG 4

/* The size of a `long long', as computed by sizeof. */
#define SIZEOF_LONG_LONG 8

/* The size of a `off_t', as computed by sizeof. */
#define SIZEOF_OFF_T 8

/* The size of a `short', as computed by sizeof. */
#define SIZEOF_SHORT 2

/* The size of a `size_t', as computed by sizeof. */
#define SIZEOF_SIZE_T 4

/* The size of a `time_t', as computed by sizeof. */
#define SIZEOF_TIME_T 8

/* The size of a `void *', as computed by sizeof. */
#define SIZEOF_VOID_P 4

/* The size of a `__int128_t', as computed by sizeof. */
#define SIZEOF___INT128_T 0

/* The size of a `__int64', as computed by sizeof. */
#define SIZEOF___INT64 0

/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1

/* define if the variable sys_errlist is declared in a system header file */
#define SYS_ERRLIST_DECLARED 

/* Define if you want to implement erts_os_hrtime() using clock_gettime() */
/* #undef SYS_HRTIME_USING_CLOCK_GETTIME */

/* Define if you want to implement erts_os_hrtime() using gethrtime() */
/* #undef SYS_HRTIME_USING_GETHRTIME */

/* Define if you want to implement erts_os_hrtime() using mach
   clock_get_time() */
/* #undef SYS_HRTIME_USING_MACH_CLOCK_GET_TIME */

/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#define TIME_WITH_SYS_TIME 1

/* Define to 1 if your <sys/time.h> declares `struct tm'. */
/* #undef TM_IN_SYS_TIME */

/* Define if you want to use dtrace for dynamic tracing */
/* #undef USE_DTRACE */

/* Define if you want to use dynamic tracing */
/* #undef USE_DYNAMIC_TRACE */

/* Define if you have matherr() function and struct exception type */
/* #undef USE_MATHERR */

/* Define if select() should be used instead of poll() */
/* #undef USE_SELECT */

/* Define if you want to use systemtap for dynamic tracing */
/* #undef USE_SYSTEMTAP */

/* Define to enable VM dynamic trace probes */
/* #undef USE_VM_PROBES */

/* Define to wall clock id to use */
#define WALL_CLOCK_ID CLOCK_REALTIME

/* Define as a string of wall clock id to use */
#define WALL_CLOCK_ID_STR "CLOCK_REALTIME"

/* Define if windows.h includes winsock2.h */
/* #undef WINDOWS_H_INCLUDES_WINSOCK2_H */

/* Define to 1 if your processor stores words with the most significant byte
   first (like Motorola and SPARC, unlike Intel and VAX). */
/* #undef WORDS_BIGENDIAN */

/* Define to empty if `const' does not conform to ANSI C. */
/* #undef const */

/* Define to `long' if <sys/types.h> does not define. */
/* #undef off_t */

/* Define to `int' if <sys/types.h> does not define. */
/* #undef pid_t */

/* Define to `unsigned' if <sys/types.h> does not define. */
/* #undef size_t */

/* Define as `fork' if `vfork' does not work. */
/* #undef vfork */


/* Redefine in6_addr. XXX this should be moved to the files where it's used? */
#ifdef HAVE_IN_ADDR6_STRUCT
#define in6_addr in_addr6
#endif
# 1245 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

/* Define a reasonable default for INADDR_LOOPBACK */
/* XXX this should be moved to the files where it's used? */
#ifdef HAVE_NO_INADDR_LOOPBACK
#define INADDR_LOOPBACK (u_long)0x7F000001
#endif
# 1251 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#ifdef REDEFINE_FD_SETSIZE
#define FD_SETSIZE 1024
#endif
# 1255 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#ifdef HAVE_GETHRVTIME_PROCFS_IOCTL
#define HAVE_GETHRVTIME
#endif
# 1259 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#if !defined(HAVE_ISFINITE) && !defined(HAVE_FINITE)
# if defined(HAVE_ISINF) && defined(HAVE_ISNAN)
#  define USE_ISINF_ISNAN
# endif
# 1264 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"
#endif
# 1265 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

#if defined(DEBUG) && defined(USE_THREADS) && !defined(ERTS_ENABLE_LOCK_CHECK)
#define ERTS_ENABLE_LOCK_CHECK 1
#endif
# 1269 "/tmp/usr/ports/lang/erlang/work/otp_src_18.3/erts/armv6-portbld-freebsd11.0/config.h"

# 26 "beam/beam_catches.h" 2
#endif
# 27 "beam/beam_catches.h"
#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 27 "beam/beam_catches.h"
# 28 "beam/beam_catches.h"
#if 0 /* expanded by -frewrite-includes */
#include "code_ix.h"
#endif /* expanded by -frewrite-includes */
# 28 "beam/beam_catches.h"
# 29 "beam/beam_catches.h"

#define BEAM_CATCHES_NIL	(-1)

void beam_catches_init(void);
void beam_catches_start_staging(void);
void beam_catches_end_staging(int commit);
unsigned beam_catches_cons(BeamInstr* cp, unsigned cdr);
BeamInstr *beam_catches_car(unsigned i);
void beam_catches_delmod(unsigned head, BeamInstr* code, unsigned code_bytes,
			 ErtsCodeIndex);

#define catch_pc(x)	beam_catches_car(catch_val((x)))

#endif	/* __BEAM_CATCHES_H */
# 43 "beam/beam_catches.h"
# 40 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "erl_thr_progress.h"
#endif /* expanded by -frewrite-includes */
# 40 "beam/beam_emu.c"
# 1 "beam/erl_thr_progress.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2013. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: Thread progress information. Used by lock free algorithms
 *              to determine when all involved threads are guaranteed to
 *              have passed a specific point of execution.
 *
 *              Usage instructions can be found in ert_thr_progress.c
 *
 * Author: 	Rickard Green
 */

#if !defined(ERL_THR_PROGRESS_H__TSD_TYPE__)
#define ERL_THR_PROGRESS_H__TSD_TYPE__

#if 0 /* expanded by -frewrite-includes */
#include "sys.h"
#endif /* expanded by -frewrite-includes */
# 34 "beam/erl_thr_progress.h"
# 35 "beam/erl_thr_progress.h"

#ifndef ERTS_SMP

#define erts_smp_thr_progress_block() ((void) 0)
#define erts_smp_thr_progress_unblock() ((void) 0)
#define erts_smp_thr_progress_is_blocking() 1

#else /* ERTS_SMP */
# 43 "beam/erl_thr_progress.h"

#define erts_smp_thr_progress_block erts_thr_progress_block
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking

void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);

typedef Uint64 ErtsThrPrgrVal;

#define ERTS_THR_PRGR_WAKEUP_DATA_SIZE 4 /* Need to be an even power of 2. */

typedef struct {
    ErtsThrPrgrVal next;
    ErtsThrPrgrVal current;
    int chk_next_ix;
    struct {
	int current;
	int waiting;
    } umrefc_ix;
} ErtsThrPrgrLeaderState;

typedef struct {
    int id;
    int is_managed;
    int is_blocking;
#ifdef ERTS_ENABLE_LOCK_CHECK
    int is_delaying; /* managed is always delaying */
#endif
# 73 "beam/erl_thr_progress.h"
    int is_temporary;

    /* --- Part below only for registered threads --- */

    ErtsThrPrgrVal wakeup_request[ERTS_THR_PRGR_WAKEUP_DATA_SIZE];

    /* --- Part below only for managed threads --- */

    int leader; /* Needs to be first in the managed threads part */
    int active;
    ErtsThrPrgrVal confirmed;
    ErtsThrPrgrLeaderState leader_state;
} ErtsThrPrgrData;

int erts_thr_progress_fatal_error_block(ErtsThrPrgrData *tmp_tpd_bufp);
void erts_thr_progress_fatal_error_wait(SWord timeout);

#endif /* ERTS_SMP */
# 91 "beam/erl_thr_progress.h"

typedef struct ErtsThrPrgrLaterOp_ ErtsThrPrgrLaterOp;
struct ErtsThrPrgrLaterOp_ {
#ifdef ERTS_SMP
    ErtsThrPrgrVal later;
#endif
# 97 "beam/erl_thr_progress.h"
    void (*func)(void *);
    void *data;
    ErtsThrPrgrLaterOp *next;
};

#endif
# 103 "beam/erl_thr_progress.h"

#if !defined(ERL_THR_PROGRESS_H__) && !defined(ERL_THR_PROGRESS_TSD_TYPE_ONLY)
#define ERL_THR_PROGRESS_H__

#if 0 /* expanded by -frewrite-includes */
#include "erl_threads.h"
#endif /* expanded by -frewrite-includes */
# 107 "beam/erl_thr_progress.h"
# 108 "beam/erl_thr_progress.h"
#if 0 /* expanded by -frewrite-includes */
#include "erl_process.h"
#endif /* expanded by -frewrite-includes */
# 108 "beam/erl_thr_progress.h"
# 109 "beam/erl_thr_progress.h"

#ifdef ERTS_SMP

/* ERTS_THR_PRGR_VAL_FIRST should only be used when initializing... */
#define ERTS_THR_PRGR_VAL_FIRST ((ErtsThrPrgrVal) 0)
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))

extern erts_tsd_key_t erts_thr_prgr_data_key__;

#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t

typedef struct {
    void *arg;
    void (*wakeup)(void *);
    void (*prepare_wait)(void *);
    void (*wait)(void *);
    void (*finalize_wait)(void *);
} ErtsThrPrgrCallbacks;

typedef struct {
    ERTS_THR_PRGR_ATOMIC current;
} ErtsThrPrgr;

typedef int ErtsThrPrgrDelayHandle;
#define ERTS_THR_PRGR_DHANDLE_MANAGED ((ErtsThrPrgrDelayHandle) -1)
/* ERTS_THR_PRGR_DHANDLE_MANAGED implies managed thread */
#define ERTS_THR_PRGR_DHANDLE_INVALID ((ErtsThrPrgrDelayHandle) -2)

extern ErtsThrPrgr erts_thr_prgr__;

void erts_thr_progress_pre_init(void);
void erts_thr_progress_init(int no_schedulers, int managed, int unmanaged);
void erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp,
					       ErtsThrPrgrCallbacks *,
					       int);
void erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *);
void erts_thr_progress_active(ErtsSchedulerData *esdp, int on);
void erts_thr_progress_wakeup(ErtsSchedulerData *esdp,
			      ErtsThrPrgrVal value);
int erts_thr_progress_update(ErtsSchedulerData *esdp);
int erts_thr_progress_leader_update(ErtsSchedulerData *esdp);
void erts_thr_progress_prepare_wait(ErtsSchedulerData *esdp);
void erts_thr_progress_finalize_wait(ErtsSchedulerData *esdp);
ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay__(void);
void erts_thr_progress_unmanaged_continue__(int umrefc_ix);

void erts_thr_progress_dbg_print_state(void);

ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);

ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
ERTS_GLB_INLINE ErtsThrPrgrDelayHandle erts_thr_progress_unmanaged_delay(void);
ERTS_GLB_INLINE void erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle);
#ifdef ERTS_ENABLE_LOCK_CHECK
ERTS_GLB_INLINE int erts_thr_progress_lc_is_delaying(void);
#endif
# 168 "beam/erl_thr_progress.h"
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current_to_later__(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(ErtsSchedulerData *);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
ERTS_GLB_INLINE int erts_thr_progress_equal(ErtsThrPrgrVal val1,
					    ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
}

ERTS_GLB_INLINE int
erts_thr_progress_is_managed_thread(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_managed;
}

ERTS_GLB_INLINE ErtsThrPrgrDelayHandle
erts_thr_progress_unmanaged_delay(void)
{
    if (erts_thr_progress_is_managed_thread())
	return ERTS_THR_PRGR_DHANDLE_MANAGED; /* Nothing to do */
    else
	return erts_thr_progress_unmanaged_delay__();
}

ERTS_GLB_INLINE void
erts_thr_progress_unmanaged_continue(ErtsThrPrgrDelayHandle handle)
{
    ASSERT(handle != ERTS_THR_PRGR_DHANDLE_MANAGED
	   || erts_thr_progress_is_managed_thread());
    if (handle != ERTS_THR_PRGR_DHANDLE_MANAGED)
	erts_thr_progress_unmanaged_continue__(handle);
}

#ifdef ERTS_ENABLE_LOCK_CHECK

ERTS_GLB_INLINE int
erts_thr_progress_lc_is_delaying(void)
{
    ErtsThrPrgrData *tpd = erts_tsd_get(erts_thr_prgr_data_key__);
    return tpd && tpd->is_delaying;
}

#endif
# 233 "beam/erl_thr_progress.h"

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current_to_later__(ErtsThrPrgrVal val)
{
    if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
	return ((ErtsThrPrgrVal) 0);
    else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
	return ((ErtsThrPrgrVal) 1);
    else
	return val + ((ErtsThrPrgrVal) 2);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_later(ErtsSchedulerData *esdp)
{
    ErtsThrPrgrData *tpd;
    ErtsThrPrgrVal val;
    if (esdp) {
	tpd = &esdp->thr_progress_data;
    managed_thread:
	val = tpd->confirmed;
	ERTS_THR_MEMORY_BARRIER;
    }
    else {
	tpd = erts_tsd_get(erts_thr_prgr_data_key__);
	if (tpd && tpd->is_managed)
	    goto managed_thread;
	val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
    }
    ASSERT(val != ERTS_THR_PRGR_VAL_WAITING);
    return erts_thr_progress_current_to_later__(val);
}

ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
    if (erts_thr_progress_is_managed_thread())
	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
    else
	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}

ERTS_GLB_INLINE int
erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
{
    if ((((((ErtsThrPrgrVal) 1) << 63) & val1)
	 ^ ((((ErtsThrPrgrVal) 1) << 63) & val0)) != 0) {
	/* May have wrapped... */
	if (val1 < (((ErtsThrPrgrVal) 1) << 62)
	    && val0 > (((ErtsThrPrgrVal) 3) << 62)) {
	    /*
	     * 'val1' has wrapped but 'val0' has not yet wrapped. While in
	     * these ranges 'current' is considered later than 'val0'.
	     */
	    return 1;
	}
    }
    return val1 > val0;
}

ERTS_GLB_INLINE int
erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
{
    if (this == val)
	return 1;
    return erts_thr_progress_has_passed__(this, val);
}

ERTS_GLB_INLINE int
erts_thr_progress_equal(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    return val1 == val2 && val1 != ERTS_THR_PRGR_INVALID;
}

ERTS_GLB_INLINE int
erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
    if (val1 == val2)
	return 0;
    if (erts_thr_progress_has_passed__(val1, val2))
	return 1;
    else
	return -1;
}	

ERTS_GLB_INLINE int
erts_thr_progress_has_reached(ErtsThrPrgrVal val)
{
    ErtsThrPrgrVal current = erts_thr_progress_current();
    return erts_thr_progress_has_reached_this(current, val);
}

#endif
# 326 "beam/erl_thr_progress.h"

#endif /* ERTS_SMP */
# 328 "beam/erl_thr_progress.h"

#endif
# 330 "beam/erl_thr_progress.h"
# 41 "beam/beam_emu.c" 2
#ifdef HIPE
#if 0 /* expanded by -frewrite-includes */
#include "hipe_mode_switch.h"
#endif /* expanded by -frewrite-includes */
# 42 "beam/beam_emu.c"
# 1 "hipe/hipe_mode_switch.h" 1
/*
 * %CopyrightBegin%

 *
 * Copyright Ericsson AB 2001-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * hipe_mode_switch.h
 */
#ifndef HIPE_MODE_SWITCH_H
#define HIPE_MODE_SWITCH_H

/* command codes for beam_emu -> hipe_mode_switch() call */
#define HIPE_MODE_SWITCH_CMD_CALL		0
#define HIPE_MODE_SWITCH_CMD_RETURN		1
#define HIPE_MODE_SWITCH_CMD_THROW		2
#define HIPE_MODE_SWITCH_CMD_RESUME		3

/* result codes for beam_emu <- hipe_mode_switch() return */
#define HIPE_MODE_SWITCH_RES_RETURN		4
#define HIPE_MODE_SWITCH_RES_CALL_EXPORTED	5
#define HIPE_MODE_SWITCH_RES_THROW		6

/* additional result codes for hipe_mode_switch() <- native return */
#define HIPE_MODE_SWITCH_RES_SUSPEND		7
#define HIPE_MODE_SWITCH_RES_WAIT		8
#define HIPE_MODE_SWITCH_RES_WAIT_TIMEOUT	9
#define HIPE_MODE_SWITCH_RES_TRAP		10

#define HIPE_MODE_SWITCH_CMD_CALL_CLOSURE	11 /* BEAM -> mode_switch */
#define HIPE_MODE_SWITCH_RES_CALL_CLOSURE	12 /* mode_switch <- native */

#define HIPE_MODE_SWITCH_RES_APPLY		13 /* mode_switch <- native */

#define HIPE_MODE_SWITCH_RES_CALL_BEAM	        14

#ifndef ASM

#if 0 /* expanded by -frewrite-includes */
#include "error.h"
#endif /* expanded by -frewrite-includes */
# 53 "hipe/hipe_mode_switch.h"
# 54 "hipe/hipe_mode_switch.h"

extern int hipe_modeswitch_debug;

void hipe_mode_switch_init(void);
void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure);
Process *hipe_mode_switch(Process*, unsigned, Eterm*);
void hipe_inc_nstack(Process *p);
void hipe_empty_nstack(Process *p);
void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free);
Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s);

ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity);
ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process*);

extern Uint hipe_beam_pc_return[];
extern Uint hipe_beam_pc_throw[];
extern Uint hipe_beam_pc_resume[];

#if ERTS_GLB_INLINE_INCL_FUNC_DEF

#if 0 /* expanded by -frewrite-includes */
#include "erl_gc.h"
#endif /* expanded by -frewrite-includes */
# 74 "hipe/hipe_mode_switch.h"
# 75 "hipe/hipe_mode_switch.h"
#if 0 /* expanded by -frewrite-includes */
#include "hipe_stack.h"
#endif /* expanded by -frewrite-includes */
# 75 "hipe/hipe_mode_switch.h"
# 1 "hipe/hipe_stack.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2012. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef HIPE_STACK_H
#define HIPE_STACK_H

#if 0 /* expanded by -frewrite-includes */
#include "hipe_arch.h"
#endif /* expanded by -frewrite-includes */
# 25 "hipe/hipe_stack.h"
# 1 "hipe/hipe_arch.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2003-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef HIPE_ARCH_H
#define HIPE_ARCH_H

extern const void *hipe_arch_primop_address(Eterm key);

/* used by beam_load.c:patch(). patchtype == am_load_fe, Value is an ErlFunEntry* */
extern void hipe_patch_address(Uint *address, Eterm patchtype, Uint value);
extern void hipe_patch_load_fe(Uint *address, Uint value);
extern int hipe_patch_insn(void *address, Uint value, Eterm type);
extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);

extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity);

#if defined(__sparc__)
#if 0 /* expanded by -frewrite-includes */
#include "hipe_sparc.h"
#endif /* expanded by -frewrite-includes */
# 37 "hipe/hipe_arch.h"
# 38 "hipe/hipe_arch.h"
#endif
# 39 "hipe/hipe_arch.h"
#if defined(__i386__)
#if 0 /* expanded by -frewrite-includes */
#include "hipe_x86.h"
#endif /* expanded by -frewrite-includes */
# 40 "hipe/hipe_arch.h"
# 41 "hipe/hipe_arch.h"
#endif
# 42 "hipe/hipe_arch.h"
#if defined(__x86_64__)
#if 0 /* expanded by -frewrite-includes */
#include "hipe_amd64.h"
#endif /* expanded by -frewrite-includes */
# 43 "hipe/hipe_arch.h"
# 44 "hipe/hipe_arch.h"
#endif
# 45 "hipe/hipe_arch.h"
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
#if 0 /* expanded by -frewrite-includes */
#include "hipe_ppc.h"
#endif /* expanded by -frewrite-includes */
# 46 "hipe/hipe_arch.h"
# 47 "hipe/hipe_arch.h"
#endif
# 48 "hipe/hipe_arch.h"
#if defined(__arm__)
#if 0 /* expanded by -frewrite-includes */
#include "hipe_arm.h"
#endif /* expanded by -frewrite-includes */
# 49 "hipe/hipe_arch.h"
# 1 "hipe/hipe_arm.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef HIPE_ARM_H
#define HIPE_ARM_H

extern void hipe_flush_icache_word(void *address);
extern void hipe_flush_icache_range(void *address, unsigned int nbytes);

/* for stack descriptor hash lookup */
#define HIPE_RA_LSR_COUNT	2	/* low 2 bits are always zero */

/* for hipe_bifs_{read,write}_{s,u}32 */
static __inline__ int hipe_word32_address_ok(void *address)
{
    return ((unsigned long)address & 0x3) == 0;
}

/* Native stack growth direction. */
#define HIPE_NSTACK_GROWS_DOWN

#define hipe_arch_name	am_arm

extern void hipe_arm_inc_stack(void);

#endif /* HIPE_ARM_H */
# 45 "hipe/hipe_arm.h"
# 50 "hipe/hipe_arch.h" 2
#endif
# 51 "hipe/hipe_arch.h"

#if !defined(AEXTERN)
#define AEXTERN(RET,NAME,PROTO)	extern RET NAME PROTO
#endif
# 55 "hipe/hipe_arch.h"

#endif /* HIPE_ARCH_H */
# 57 "hipe/hipe_arch.h"
# 26 "hipe/hipe_stack.h" 2

/*
 * Stack descriptors.
 */

#if 0 /* expanded by -frewrite-includes */
#include <stddef.h>	/* offsetof() */
#endif /* expanded by -frewrite-includes */
# 31 "hipe/hipe_stack.h"
# 32 "hipe/hipe_stack.h"

struct sdesc {
    struct {
	unsigned long hvalue;	/* return address */
	struct sdesc *next;	/* hash collision chain */
    } bucket;
    unsigned int summary; /* frame size, exn handler presence flag, arity */
#ifdef DEBUG
    Eterm dbg_M, dbg_F;
    unsigned dbg_A;
#endif
# 43 "hipe/hipe_stack.h"
    unsigned int livebits[1]; /* size depends on arch & data in summary field */
};

struct sdesc_with_exnra {
    unsigned long exnra;
    struct sdesc sdesc;
};

static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc)
{
    return sdesc->summary >> 9;
}

static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc)
{
    return sdesc->summary & 0xFF;
}

static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc)
{
    if ((sdesc->summary & (1<<8))) {
	const char *tmp;
	tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc);
	return ((const struct sdesc_with_exnra*)tmp)->exnra;
    }
    return 0;
}

struct hipe_sdesc_table {
    unsigned int log2size;
    unsigned int mask;		/* INV: mask == (1 << log2size)-1 */
    unsigned int used;
    struct sdesc **bucket;
};
extern struct hipe_sdesc_table hipe_sdesc_table;

extern struct sdesc *hipe_put_sdesc(struct sdesc*);
extern void hipe_init_sdesc_table(struct sdesc*);
extern struct sdesc *hipe_decode_sdesc(Eterm);

#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
#define __builtin_expect(x, expected_value) (x)
#endif
# 86 "hipe/hipe_stack.h"
#define likely(x)	__builtin_expect((x),1)
#define unlikely(x)	__builtin_expect((x),0)

static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra)
{
    unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
    const struct sdesc *sdesc = hipe_sdesc_table.bucket[i];
    if (likely(sdesc->bucket.hvalue == ra))
	return sdesc;
    do {
	sdesc = sdesc->bucket.next;
    } while (sdesc->bucket.hvalue != ra);
    return sdesc;
}

AEXTERN(void,nbif_stack_trap_ra,(void));

extern void hipe_print_nstack(Process*);
extern void hipe_find_handler(Process*);
extern void (*hipe_handle_stack_trap(Process*))(void);
extern void hipe_update_stack_trap(Process*, const struct sdesc*);
extern int hipe_fill_stacktrace(Process*, int, Eterm**);

#if 0 && defined(HIPE_NSTACK_GROWS_UP)
#define hipe_nstack_start(p)	((p)->hipe.nstack)
#define hipe_nstack_used(p)	((p)->hipe.nsp - (p)->hipe.nstack)
#define hipe_nstack_avail(p)	((p)->hipe.nstend - (p)->hipe.nsp)
#endif
# 114 "hipe/hipe_stack.h"
#if defined(HIPE_NSTACK_GROWS_DOWN)
#define hipe_nstack_start(p)	((p)->hipe.nsp)
#define hipe_nstack_used(p)	((p)->hipe.nstend - (p)->hipe.nsp)
#define hipe_nstack_avail(p)	((unsigned)((p)->hipe.nsp - (p)->hipe.nstack))
#endif
# 119 "hipe/hipe_stack.h"

/* ensure that at least nwords words are available on the native stack */
static __inline__ void hipe_check_nstack(Process *p, unsigned nwords)
{
    extern void hipe_inc_nstack(Process *p);

    while (hipe_nstack_avail(p) < nwords)
	hipe_inc_nstack(p);
}

/*
 * GC support procedures
 */
extern Eterm *fullsweep_nstack(Process *p, Eterm *n_htop);
extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop);

#endif /* HIPE_STACK_H */
# 136 "hipe/hipe_stack.h"
# 76 "hipe/hipe_mode_switch.h" 2

#if defined(__sparc__)
#if 0 /* expanded by -frewrite-includes */
#include "hipe_sparc_glue.h"
#endif /* expanded by -frewrite-includes */
# 78 "hipe/hipe_mode_switch.h"
# 79 "hipe/hipe_mode_switch.h"
#elif defined(__i386__)
# 80 "hipe/hipe_mode_switch.h"
#if 0 /* expanded by -frewrite-includes */
#include "hipe_x86_glue.h"
#endif /* expanded by -frewrite-includes */
# 80 "hipe/hipe_mode_switch.h"
# 81 "hipe/hipe_mode_switch.h"
#elif defined(__x86_64__)
# 82 "hipe/hipe_mode_switch.h"
#if 0 /* expanded by -frewrite-includes */
#include "hipe_amd64_glue.h"
#endif /* expanded by -frewrite-includes */
# 82 "hipe/hipe_mode_switch.h"
# 83 "hipe/hipe_mode_switch.h"
#elif defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
# 84 "hipe/hipe_mode_switch.h"
#if 0 /* expanded by -frewrite-includes */
#include "hipe_ppc_glue.h"
#endif /* expanded by -frewrite-includes */
# 84 "hipe/hipe_mode_switch.h"
# 85 "hipe/hipe_mode_switch.h"
#elif defined(__arm__)
# 86 "hipe/hipe_mode_switch.h"
#if 0 /* expanded by -frewrite-includes */
#include "hipe_arm_glue.h"
#endif /* expanded by -frewrite-includes */
# 86 "hipe/hipe_mode_switch.h"
# 1 "hipe/hipe_arm_glue.h" 1
/*
 * %CopyrightBegin%
 * 
 * Copyright Ericsson AB 2005-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef HIPE_ARM_GLUE_H
#define HIPE_ARM_GLUE_H

#if 0 /* expanded by -frewrite-includes */
#include "hipe_arm_asm.h"		/* for NR_ARG_REGS, ARM_LEAF_WORDS */
#endif /* expanded by -frewrite-includes */
# 25 "hipe/hipe_arm_glue.h"
# 1 "armv6-portbld-freebsd11.0/opt/smp/hipe_arm_asm.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2005-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef HIPE_ARM_ASM_H
#define HIPE_ARM_ASM_H

/*
 * Tunables.
 */

#define ARM_LEAF_WORDS	16
#define ARM_NR_ARG_REGS	3
#define NR_ARG_REGS	3


#ifdef ASM
/*
 * Only assembler stuff from here on (when included from *.S)
 */

/*
 * Reserved registers.
 */
#define P	r11
#define NSP	r10
#define HP	r9
#define TEMP_LR	r8

/*
 * Context switching macros.
 *
 * RESTORE_CONTEXT and RESTORE_CONTEXT_QUICK do not affect
 * the condition register.
 */
#define SAVE_CONTEXT_QUICK	\
	mov	TEMP_LR, lr

#define RESTORE_CONTEXT_QUICK	\
	mov	lr, TEMP_LR

#define SAVE_CACHED_STATE	\
	str	HP, [P, #P_HP];	\
	str	NSP, [P, #P_NSP]

#define RESTORE_CACHED_STATE	\
	ldr	HP, [P, #P_HP];	\
	ldr	NSP, [P, #P_NSP]

#define SAVE_CONTEXT_BIF	\
	mov	TEMP_LR, lr;	\
	str	HP, [P, #P_HP]

#define RESTORE_CONTEXT_BIF	\
	ldr	HP, [P, #P_HP]

#define SAVE_CONTEXT_GC	\
	mov	TEMP_LR, lr;	\
	str	lr, [P, #P_NRA];	\
	str	NSP, [P, #P_NSP];	\
	str	HP, [P, #P_HP]

#define RESTORE_CONTEXT_GC	\
	ldr	HP, [P, #P_HP]

/*
 * Argument (parameter) registers.
 */


#define ARG0	r1
#define ARG1	r2
#define ARG2	r3

/*
 * TEMP_ARG0:
 *	Used in nbif_stack_trap_ra to preserve the return value.
 *	Must be a C callee-save register.
 *	Must be otherwise unused in the return path.
 *
 * TEMP_ARG0:
 *	Used in hipe_arm_inc_stack to preserve the return address
 *	(TEMP_LR contains the caller's saved return address).
 *	Must be a C callee-save register.
 *	Must be otherwise unused in the call path.
 */
#define TEMP_ARG0	r7


#define LOAD_ARG_REGS	ldr r1, [P, #P_ARG0] ; ldr r2, [P, #P_ARG1] ; ldr r3, [P, #P_ARG2] ; 

#define STORE_ARG_REGS	str r1, [P, #P_ARG0] ; str r2, [P, #P_ARG1] ; str r3, [P, #P_ARG2] ; 


/* #define NBIF_ARG_1_0	# mov	r1, r1 */
/* #define NBIF_ARG_2_0	# mov	r1, r1 */
/* #define NBIF_ARG_2_1	# mov	r2, r2 */
/* #define NBIF_ARG_3_0	# mov	r1, r1 */
/* #define NBIF_ARG_3_1	# mov	r2, r2 */
/* #define NBIF_ARG_3_2	# mov	r3, r3 */
/* #define NBIF_ARG_4_0	# mov	r1, r1 */
/* #define NBIF_ARG_4_1	# mov	r2, r2 */
/* #define NBIF_ARG_4_2	# mov	r3, r3 */
/* #define NBIF_ARG_4_3	ldr	r4, [NSP, #0] */
/* #define NBIF_ARG_5_0	# mov	r1, r1 */
/* #define NBIF_ARG_5_1	# mov	r2, r2 */
/* #define NBIF_ARG_5_2	# mov	r3, r3 */
/* #define NBIF_ARG_5_3	ldr	r4, [NSP, #4] */
/* #define NBIF_ARG_5_4	ldr	r5, [NSP, #0] */

/* #define NBIF_RET_0	mov pc, TEMP_LR */
/* #define NBIF_RET_1	mov pc, TEMP_LR */
/* #define NBIF_RET_2	mov pc, TEMP_LR */
/* #define NBIF_RET_3	mov pc, TEMP_LR */
/* #define NBIF_RET_4	add	NSP, NSP, #4
	mov pc, TEMP_LR */
/* #define NBIF_RET_5	add	NSP, NSP, #8
	mov pc, TEMP_LR */

/* #define QUICK_CALL_RET_F_0 b F */
/* #define QUICK_CALL_RET_F_1 b F */
/* #define QUICK_CALL_RET_F_2 b F */
/* #define QUICK_CALL_RET_F_3 b F */
/* #define QUICK_CALL_RET_F_5 add NSP, NSP, #8 ; b F */

#endif /* ASM */
# 144 "armv6-portbld-freebsd11.0/opt/smp/hipe_arm_asm.h"

#endif /* HIPE_ARM_ASM_H */
# 146 "armv6-portbld-freebsd11.0/opt/smp/hipe_arm_asm.h"
# 26 "hipe/hipe_arm_glue.h" 2
#define NR_LEAF_WORDS			ARM_LEAF_WORDS
#define HIPE_ARCH_CALL_TO_NATIVE	hipe_arm_call_to_native
#define HIPE_ARCH_RETURN_TO_NATIVE	hipe_arm_return_to_native
#define HIPE_ARCH_TAILCALL_TO_NATIVE	hipe_arm_tailcall_to_native
#define HIPE_ARCH_THROW_TO_NATIVE	hipe_arm_throw_to_native
#if 0 /* expanded by -frewrite-includes */
#include "hipe_risc_glue.h"
#endif /* expanded by -frewrite-includes */
# 31 "hipe/hipe_arm_glue.h"
# 1 "hipe/hipe_risc_glue.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2008-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */


#ifndef HIPE_RISC_GLUE_H
#define HIPE_RISC_GLUE_H

/* arch wrapper does:
 * #include "hipe_${arch}_asm.h"	// for NR_ARG_REGS, ${ARCH}_LEAF_WORDS
 * #define NR_LEAF_WORDS		${ARCH}_LEAF_WORDS
 * #define HIPE_ARCH_CALL_TO_NATIVE	hipe_${arch}_call_to_native
 * #define HIPE_ARCH_RETURN_TO_NATIVE	hipe_${arch}_return_to_native
 * #define HIPE_ARCH_TAILCALL_TO_NATIVE	hipe_${arch}_tailcall_to_native
 * #define HIPE_ARCH_THROW_TO_NATIVE	hipe_${arch}_throw_to_native
 * #include "hipe_risc_glue.h"
 */

/* Emulated code recursively calls native code.
   The return address is `nbif_return', which is exported so that
   tailcalls from native to emulated code can be identified. */
unsigned int HIPE_ARCH_CALL_TO_NATIVE(Process*);
AEXTERN(void,nbif_return,(void));

/* Native-mode stubs for calling emulated-mode closures. */
AEXTERN(void,nbif_ccallemu0,(void));
AEXTERN(void,nbif_ccallemu1,(void));
AEXTERN(void,nbif_ccallemu2,(void));
AEXTERN(void,nbif_ccallemu3,(void));
AEXTERN(void,nbif_ccallemu4,(void));
AEXTERN(void,nbif_ccallemu5,(void));
AEXTERN(void,nbif_ccallemu6,(void));

/* Default exception handler for native code. */
AEXTERN(void,nbif_fail,(void));

/* Emulated code returns to its native code caller. */
unsigned int HIPE_ARCH_RETURN_TO_NATIVE(Process*);

/* Emulated code tailcalls native code. */
unsigned int HIPE_ARCH_TAILCALL_TO_NATIVE(Process*);

/* Emulated code throws an exception to its native code caller. */
unsigned int HIPE_ARCH_THROW_TO_NATIVE(Process*);

static __inline__ unsigned int max(unsigned int x, unsigned int y)
{
    return (x > y) ? x : y;
}

static __inline__ void hipe_arch_glue_init(void)
{
    static struct sdesc_with_exnra nbif_return_sdesc = {
	.exnra = (unsigned long)&nbif_fail,
	.sdesc = {
	    .bucket = { .hvalue = (unsigned long)&nbif_return },
	    .summary = (1<<8),
	},
    };
    hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}

static __inline__ void hipe_push_risc_nra_frame(Process *p)
{
    p->hipe.nsp -= 1;
    p->hipe.nsp[0] = (Eterm)p->hipe.nra;
}

static __inline__ void hipe_pop_risc_nra_frame(Process *p)
{
    p->hipe.nra = (void(*)(void))p->hipe.nsp[0];
    p->hipe.nsp += 1;
}

/* PRE: arity <= NR_ARG_REGS */
static __inline__ void
hipe_write_risc_regs(Process *p, unsigned int arity, Eterm reg[])
{
#if NR_ARG_REGS > 0
    int i;
    for (i = arity; --i >= 0;)
	p->def_arg_reg[i] = reg[i];
#endif
# 100 "hipe/hipe_risc_glue.h"
}

/* PRE: arity <= NR_ARG_REGS */
static __inline__ void
hipe_read_risc_regs(Process *p, unsigned int arity, Eterm reg[])
{
#if NR_ARG_REGS > 0
    int i;
    for (i = arity; --i >= 0;)
	reg[i] = p->def_arg_reg[i];
#endif
# 111 "hipe/hipe_risc_glue.h"
}

static __inline__ void
hipe_push_risc_params(Process *p, unsigned int arity, Eterm reg[])
{
    unsigned int i;

    i = arity;
    if (i > NR_ARG_REGS) {
	Eterm *nsp = p->hipe.nsp;
	i = NR_ARG_REGS;
	do {
	    *--nsp = reg[i++];
	} while (i < arity);
	p->hipe.nsp = nsp;
	i = NR_ARG_REGS;
    }
    /* INV: i <= NR_ARG_REGS */
    hipe_write_risc_regs(p, i, reg);
}

static __inline__ void
hipe_pop_risc_params(Process *p, unsigned int arity, Eterm reg[])
{
    unsigned int i;

    i = arity;
    if (i > NR_ARG_REGS) {
	Eterm *nsp = p->hipe.nsp;
	do {
	    reg[--i] = *nsp++;
	} while (i > NR_ARG_REGS);
	p->hipe.nsp = nsp;
	/* INV: i == NR_ARG_REGS */
    }
    /* INV: i <= NR_ARG_REGS */
    hipe_read_risc_regs(p, i, reg);
}

/* BEAM recursively calls native code. */
static __inline__ unsigned int
hipe_call_to_native(Process *p, unsigned int arity, Eterm reg[])
{
    int nstkargs;

    if ((nstkargs = arity - NR_ARG_REGS) < 0)
	nstkargs = 0;
    hipe_check_nstack(p, max(nstkargs + 1, NR_LEAF_WORDS));
    hipe_push_risc_nra_frame(p);			/* needs 1 word */
    hipe_push_risc_params(p, arity, reg);	/* needs nstkargs words */
    return HIPE_ARCH_CALL_TO_NATIVE(p);
}

/* Native called BEAM, which now tailcalls native. */
static __inline__ unsigned int
hipe_tailcall_to_native(Process *p, unsigned int arity, Eterm reg[])
{
    int nstkargs;

    if ((nstkargs = arity - NR_ARG_REGS) < 0)
	nstkargs = 0;
    hipe_check_nstack(p, max(nstkargs, NR_LEAF_WORDS));
    hipe_push_risc_params(p, arity, reg);	/* needs nstkargs words */
    return HIPE_ARCH_TAILCALL_TO_NATIVE(p);
}

/* BEAM called native, which has returned. Clean up. */
static __inline__ void hipe_return_from_native(Process *p)
{
    hipe_pop_risc_nra_frame(p);
}

/* BEAM called native, which has thrown an exception. Clean up. */
static __inline__ void hipe_throw_from_native(Process *p)
{
    hipe_pop_risc_nra_frame(p);
}

/* BEAM called native, which now calls BEAM.
   Move the parameters to reg[].
   Return zero if this is a tailcall, non-zero if the call is recursive.
   If tailcall, also clean up native stub continuation. */
static __inline__ int
hipe_call_from_native_is_recursive(Process *p, Eterm reg[])
{
    hipe_pop_risc_params(p, p->arity, reg);
    if (p->hipe.nra != (void(*)(void))&nbif_return)
	return 1;
    hipe_pop_risc_nra_frame(p);
    return 0;
}

/* BEAM called native, which called BIF that returned trap
 * Discard bif parameters.
 * If tailcall, also clean up native stub continuation. */
static __inline__ int
hipe_trap_from_native_is_recursive(Process *p)
{
    if (p->hipe.narity > NR_ARG_REGS) {
	p->hipe.nsp += (p->hipe.narity - NR_ARG_REGS);
    }
    if (p->hipe.nra != (void(*)(void))&nbif_return)
	return 1;
    hipe_pop_risc_nra_frame(p);
    return 0;
}

/* Native called BIF. Is it a recursive call?
   i.e should we return back to native when BIF is done? */
static __inline__ int
hipe_bifcall_from_native_is_recursive(Process *p)
{
    return (p->hipe.nra != (void(*)(void))&nbif_return);
}


/* Native makes a call which needs to unload the parameters.
   This differs from hipe_call_from_native_is_recursive() in
   that it doesn't check for or pop the BEAM-calls-native frame.
   It's currently only used in the implementation of apply. */
static __inline__ void
hipe_pop_params(Process *p, unsigned int arity, Eterm reg[])
{
    hipe_pop_risc_params(p, arity, reg);
}

/* Native called BEAM, which now returns back to native. */
static __inline__ unsigned int hipe_return_to_native(Process *p)
{
    return HIPE_ARCH_RETURN_TO_NATIVE(p);
}

/* Native called BEAM, which now throws an exception back to native. */
static __inline__ unsigned int hipe_throw_to_native(Process *p)
{
    return HIPE_ARCH_THROW_TO_NATIVE(p);
}

/* Return the address of a stub switching a native closure call to BEAM. */
static __inline__ const void *hipe_closure_stub_address(unsigned int arity)
{
#if NR_ARG_REGS == 0
    return &nbif_ccallemu0;
#else	/* > 0 */
# 255 "hipe/hipe_risc_glue.h"
    switch (arity) {
      case 0:	return &nbif_ccallemu0;
#if NR_ARG_REGS == 1
      default:	return &nbif_ccallemu1;
#else	/* > 1 */
# 260 "hipe/hipe_risc_glue.h"
      case 1:	return &nbif_ccallemu1;
#if NR_ARG_REGS == 2
      default:	return &nbif_ccallemu2;
#else	/* > 2 */
# 264 "hipe/hipe_risc_glue.h"
      case 2:	return &nbif_ccallemu2;
#if NR_ARG_REGS == 3
      default:	return &nbif_ccallemu3;
#else	/* > 3 */
# 268 "hipe/hipe_risc_glue.h"
      case 3:	return &nbif_ccallemu3;
#if NR_ARG_REGS == 4
      default:	return &nbif_ccallemu4;
#else	/* > 4 */
# 272 "hipe/hipe_risc_glue.h"
      case 4:	return &nbif_ccallemu4;
#if NR_ARG_REGS == 5
      default:	return &nbif_ccallemu5;
#else	/* > 5 */
# 276 "hipe/hipe_risc_glue.h"
      case 5:	return &nbif_ccallemu5;
#if NR_ARG_REGS == 6
      default:	return &nbif_ccallemu6;
#else
# 280 "hipe/hipe_risc_glue.h"
#error "NR_ARG_REGS > 6 NOT YET IMPLEMENTED"
#endif	/* > 6 */
# 282 "hipe/hipe_risc_glue.h"
#endif	/* > 5 */
# 283 "hipe/hipe_risc_glue.h"
#endif	/* > 4 */
# 284 "hipe/hipe_risc_glue.h"
#endif	/* > 3 */
# 285 "hipe/hipe_risc_glue.h"
#endif	/* > 2 */
# 286 "hipe/hipe_risc_glue.h"
#endif	/* > 1 */
# 287 "hipe/hipe_risc_glue.h"
    }
#endif	/* > 0 */
# 289 "hipe/hipe_risc_glue.h"
}

#endif /* HIPE_RISC_GLUE_H */
# 292 "hipe/hipe_risc_glue.h"
# 32 "hipe/hipe_arm_glue.h" 2

#endif /* HIPE_ARM_GLUE_H */
# 34 "hipe/hipe_arm_glue.h"
# 87 "hipe/hipe_mode_switch.h" 2
#endif
# 88 "hipe/hipe_mode_switch.h"

extern Eterm hipe_beam_catch_throw;

ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
{
    if (!hipe_bifcall_from_native_is_recursive(p))
	return;

    /* ensure that at least 2 words are available on the BEAM stack */
    if ((p->stop - 2) < p->htop) {
	p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
	ASSERT(!((p->stop - 2) < p->htop));
    }
    p->stop -= 2;
    p->stop[0] = NIL;
    p->stop[1] = hipe_beam_catch_throw;
}

ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process *p)
{
    if (!hipe_bifcall_from_native_is_recursive(p))
	return;

    ASSERT(p->stop[0] == NIL && p->stop[1] == hipe_beam_catch_throw);
    p->stop += 2;
}

#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
# 116 "hipe/hipe_mode_switch.h"

#endif	/* ASM */
# 118 "hipe/hipe_mode_switch.h"

#endif /* HIPE_MODE_SWITCH_H */
# 120 "hipe/hipe_mode_switch.h"
# 43 "beam/beam_emu.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "hipe_bif1.h"
#endif /* expanded by -frewrite-includes */
# 43 "beam/beam_emu.c"
# 1 "hipe/hipe_bif1.h" 1
/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2001-2011. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */
/*
 * hipe_bif1.h
 *
 * Performance analysis support.
 */
#ifndef HIPE_BIF1_H
#define HIPE_BIF1_H

struct hipe_call_count {
    unsigned count;
    Uint opcode;
};

extern unsigned int hipe_trap_count;

#endif /* HIPE_BIF1_H */
# 36 "hipe/hipe_bif1.h"
# 44 "beam/beam_emu.c" 2
#endif
# 45 "beam/beam_emu.c"
#if 0 /* expanded by -frewrite-includes */
#include "dtrace-wrapper.h"
#endif /* expanded by -frewrite-includes */
# 45 "beam/beam_emu.c"
# 46 "beam/beam_emu.c"

/* #define HARDDEBUG 1 */

#if defined(NO_JUMP_TABLE)
#  define OpCase(OpCode)    case op_##OpCode
#  define CountCase(OpCode) case op_count_##OpCode
#  define OpCode(OpCode)    ((Uint*)op_##OpCode)
#  define Goto(Rel) {Go = (int)(UWord)(Rel); goto emulator_loop;}
#  define LabelAddr(Addr) &&##Addr
#else
# 56 "beam/beam_emu.c"
#  define OpCase(OpCode)    lb_##OpCode
#  define CountCase(OpCode) lb_count_##OpCode
#  define Goto(Rel) goto *((void *)Rel)
#  define LabelAddr(Label) &&Label
#  define OpCode(OpCode)  (&&lb_##OpCode)
#endif
# 62 "beam/beam_emu.c"

#ifdef ERTS_ENABLE_LOCK_CHECK
#  ifdef ERTS_SMP
#    define PROCESS_MAIN_CHK_LOCKS(P)					\
do {									\
    if ((P)) {								\
	erts_proc_lc_chk_only_proc_main((P));				\
    }									\
    else								\
	erts_lc_check_exact(NULL, 0);					\
    	ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking());		\
} while (0)
#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \
        if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\
					   __FILE__, __LINE__)
#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \
        if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN)
#  else
# 80 "beam/beam_emu.c"
#    define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
#    define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
#    define PROCESS_MAIN_CHK_LOCKS(P) erts_lc_check_exact(NULL, 0)
#  endif
# 84 "beam/beam_emu.c"
#else
# 85 "beam/beam_emu.c"
#  define PROCESS_MAIN_CHK_LOCKS(P)
#  define ERTS_SMP_REQ_PROC_MAIN_LOCK(P)
#  define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P)
#endif
# 89 "beam/beam_emu.c"

/*
 * Define macros for deep checking of terms.
 */

#if defined(HARDDEBUG)

#  define CHECK_TERM(T) size_object(T)

#  define CHECK_ARGS(PC)                 \
do {                                     \
  int i_;                                \
  int Arity_ = PC[-1];                   \
  if (Arity_ > 0) {                      \
	CHECK_TERM(r(0));                \
  }                                      \
  for (i_ = 1; i_ < Arity_; i_++) {      \
	CHECK_TERM(x(i_));               \
  }                                      \
} while (0)
    
#else
# 111 "beam/beam_emu.c"
#  define CHECK_TERM(T) ASSERT(!is_CP(T))
#  define CHECK_ARGS(T)
#endif
# 114 "beam/beam_emu.c"

#ifndef MAX
#define MAX(x, y) (((x) > (y)) ? (x) : (y))
#endif
# 118 "beam/beam_emu.c"

#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4]))
#define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))


/*
 * We reuse some of fields in the save area in the process structure.
 * This is safe to do, since this space is only activly used when
 * the process is switched out.
 */
#define REDS_IN(p)  ((p)->def_arg_reg[5])

/*
 * Add a byte offset to a pointer to Eterm.  This is useful when the
 * the loader has precalculated a byte offset.
 */
#define ADD_BYTE_OFFSET(ptr, offset) \
   ((Eterm *) (((unsigned char *)ptr) + (offset)))

/* We don't check the range if an ordinary switch is used */
#ifdef NO_JUMP_TABLE
#define VALID_INSTR(IP) ((UWord)(IP) < (NUMBER_OF_OPCODES*2+10))
#else
# 141 "beam/beam_emu.c"
#define VALID_INSTR(IP) \
   ((SWord)LabelAddr(emulator_loop) <= (SWord)(IP) && \
    (SWord)(IP) < (SWord)LabelAddr(end_emulator_loop))
#endif /* NO_JUMP_TABLE */
# 145 "beam/beam_emu.c"

#define SET_CP(p, ip)           \
   ASSERT(VALID_INSTR(*(ip)));  \
   (p)->cp = (ip)

#define SET_I(ip) \
   ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
   I = (ip)

#define FetchArgs(S1, S2) tmp_arg1 = (S1); tmp_arg2 = (S2)

/*
 * Store a result into a register given a destination descriptor.
 */

#define StoreResult(Result, DestDesc)               \
  do {                                              \
    Eterm stb_reg;                                  \
    stb_reg = (DestDesc);                           \
    CHECK_TERM(Result);                             \
    switch (beam_reg_tag(stb_reg)) {                \
    case R_REG_DEF:                                 \
      r(0) = (Result); break;                       \
    case X_REG_DEF:                                 \
      xb(x_reg_offset(stb_reg)) = (Result); break;  \
    default:                                        \
      yb(y_reg_offset(stb_reg)) = (Result); break;  \
    }                                               \
  } while (0)

#define StoreSimpleDest(Src, Dest) Dest = (Src)

/*
 * Store a result into a register and execute the next instruction.
 * Dst points to the word with a destination descriptor, which MUST
 * be just before the next instruction.
 */
 
#define StoreBifResult(Dst, Result)                          \
  do {                                                       \
    BeamInstr* stb_next;                                         \
    Eterm stb_reg;                                           \
    stb_reg = Arg(Dst);                                      \
    I += (Dst) + 2;                                          \
    stb_next = (BeamInstr *) *I;                                 \
    CHECK_TERM(Result);                                      \
    switch (beam_reg_tag(stb_reg)) {                         \
    case R_REG_DEF:                                          \
      r(0) = (Result); Goto(stb_next);                       \
    case X_REG_DEF:                                          \
      xb(x_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
    default:                                                 \
      yb(y_reg_offset(stb_reg)) = (Result); Goto(stb_next);  \
    }                                                        \
  } while (0)

#define ClauseFail() goto jump_f

#define SAVE_CP(X)				\
   do {						\
      *(X) = make_cp(c_p->cp);			\
      c_p->cp = 0;				\
   } while(0)

#define RESTORE_CP(X)		SET_CP(c_p, (BeamInstr *) cp_val(*(X)))

#define ISCATCHEND(instr) ((Eterm *) *(instr) == OpCode(catch_end_y))

/*
 * Special Beam instructions.
 */

BeamInstr beam_apply[2];
BeamInstr beam_exit[1];
BeamInstr beam_continue_exit[1];

BeamInstr* em_call_error_handler;
BeamInstr* em_apply_bif;
BeamInstr* em_call_nif;


/* NOTE These should be the only variables containing trace instructions.
**      Sometimes tests are form the instruction value, and sometimes
**      for the refering variable (one of these), and rouge references
**      will most likely cause chaos.
*/
BeamInstr beam_return_to_trace[1];   /* OpCode(i_return_to_trace) */
BeamInstr beam_return_trace[1];      /* OpCode(i_return_trace) */
BeamInstr beam_exception_trace[1];   /* UGLY also OpCode(i_return_trace) */
BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */


/*
 * All Beam instructions in numerical order.
 */

#ifndef NO_JUMP_TABLE
void** beam_ops;
#endif
# 244 "beam/beam_emu.c"

#define SWAPIN             \
    HTOP = HEAP_TOP(c_p);  \
    E = c_p->stop

#define SWAPOUT            \
    HEAP_TOP(c_p) = HTOP;  \
    c_p->stop = E

/*
 * Use LIGHT_SWAPOUT when the called function
 * will call HeapOnlyAlloc() (and never HAlloc()).
 */
#ifdef DEBUG
#  /* The stack pointer is used in an assertion. */
#  define LIGHT_SWAPOUT SWAPOUT
#else
# 261 "beam/beam_emu.c"
#  define LIGHT_SWAPOUT HEAP_TOP(c_p) = HTOP
#endif
# 263 "beam/beam_emu.c"

/*
 * Use LIGHT_SWAPIN when we know that c_p->stop cannot
 * have been updated (i.e. if there cannot have been
 * a garbage-collection).
 */

#define LIGHT_SWAPIN HTOP = HEAP_TOP(c_p)

#ifdef FORCE_HEAP_FRAGS
#  define HEAP_SPACE_VERIFIED(Words) do { \
      c_p->space_verified = (Words);	  \
      c_p->space_verified_from = HTOP;	  \
    }while(0)
#else
# 278 "beam/beam_emu.c"
#  define HEAP_SPACE_VERIFIED(Words) ((void)0)
#endif
# 280 "beam/beam_emu.c"

#define PRE_BIF_SWAPOUT(P)						\
     HEAP_TOP((P)) = HTOP;  						\
     (P)->stop = E;  							\
     PROCESS_MAIN_CHK_LOCKS((P));					\
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK((P))

#define db(N) (N)
#define tb(N) (N)
#define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N)))
#define yb(N) (*(Eterm *) (((unsigned char *)E) + (N)))
#define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
#define Qb(N) (N)
#define Ib(N) (N)
#define x(N) reg[N]
#define y(N) E[N]
#define r(N) x##N

/*
 * Makes sure that there are StackNeed + HeapNeed + 1 words available
 * on the combined heap/stack segment, then allocates StackNeed + 1
 * words on the stack and saves CP.
 *
 * M is number of live registers to preserve during garbage collection
 */

#define AH(StackNeed, HeapNeed, M) \
  do { \
     int needed; \
     needed = (StackNeed) + 1; \
     if (E - HTOP < (needed + (HeapNeed))) { \
           SWAPOUT; \
           reg[0] = r(0); \
           PROCESS_MAIN_CHK_LOCKS(c_p); \
           FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
           ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
           PROCESS_MAIN_CHK_LOCKS(c_p); \
           r(0) = reg[0]; \
           SWAPIN; \
     } \
     E -= needed; \
     SAVE_CP(E); \
  } while (0)

#define Allocate(Ns, Live) AH(Ns, 0, Live)

#define AllocateZero(Ns, Live)             \
 do { Eterm* ptr;                          \
      int i = (Ns);                        \
      AH(i, 0, Live);                      \
      for (ptr = E + i; ptr > E; ptr--) {  \
	 make_blank(*ptr);                 \
     }                                     \
  } while (0)

#define AllocateHeap(Ns, Nh, Live) AH(Ns, Nh, Live)

#define AllocateHeapZero(Ns, Nh, Live)     \
 do { Eterm* ptr;                          \
      int i = (Ns);                        \
      AH(i, Nh, Live);                     \
      for (ptr = E + i; ptr > E; ptr--) {  \
	 make_blank(*ptr);                 \
     }                                     \
  } while (0)

#define AllocateInit(Ns, Live, Y) \
  do { AH(Ns, 0, Live); make_blank(Y); } while (0)

/*
 * Like the AH macro, but allocates no additional heap space.
 */

#define A(StackNeed, M) AH(StackNeed, 0, M)

#define D(N)             \
     RESTORE_CP(E);      \
     E += (N) + 1;



#define TestBinVHeap(VNh, Nh, Live)                             		\
  do {                                                          		\
    unsigned need = (Nh);                                       		\
    if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
       SWAPOUT;                                                 		\
       reg[0] = r(0);                                           		\
       PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  		\
       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);					\
       PROCESS_MAIN_CHK_LOCKS(c_p);                             		\
       r(0) = reg[0];                                           		\
       SWAPIN;                                                  		\
    }                                                           		\
    HEAP_SPACE_VERIFIED(need);                                                  \
  } while (0)



/*
 * Check if Nh words of heap are available; if not, do a garbage collection.
 * Live is number of active argument registers to be preserved.
 */

#define TestHeap(Nh, Live)                                      \
  do {                                                          \
    unsigned need = (Nh);                                       \
    if (E - HTOP < need) {                                      \
       SWAPOUT;                                                 \
       reg[0] = r(0);                                           \
       PROCESS_MAIN_CHK_LOCKS(c_p);                             \
       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live));  \
       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);			\
       PROCESS_MAIN_CHK_LOCKS(c_p);                             \
       r(0) = reg[0];                                           \
       SWAPIN;                                                  \
    }                                                           \
    HEAP_SPACE_VERIFIED(need);                             \
  } while (0)

/*
 * Check if Nh words of heap are available; if not, do a garbage collection.
 * Live is number of active argument registers to be preserved.
 * Takes special care to preserve Extra if a garbage collection occurs.
 */

#define TestHeapPreserve(Nh, Live, Extra)				\
  do {									\
    unsigned need = (Nh);						\
    if (E - HTOP < need) {						\
       SWAPOUT;								\
       reg[0] = r(0);							\
       reg[Live] = Extra;						\
       PROCESS_MAIN_CHK_LOCKS(c_p);					\
       FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1);	\
       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);				\
       PROCESS_MAIN_CHK_LOCKS(c_p);					\
       if (Live > 0) {							\
	   r(0) = reg[0];						\
       }								\
       Extra = reg[Live];						\
       SWAPIN;								\
    }									\
    HEAP_SPACE_VERIFIED(need);                                      \
  } while (0)

#define TestHeapPutList(Need, Reg)		\
  do {						\
     TestHeap((Need), 1);			\
     PutList(Reg, r(0), r(0), StoreSimpleDest);	\
     CHECK_TERM(r(0));				\
  } while (0)

#define Init(N) make_blank(yb(N))

#define Init2(Y1, Y2) do { make_blank(Y1); make_blank(Y2); } while (0)
#define Init3(Y1, Y2, Y3) \
   do { make_blank(Y1); make_blank(Y2); make_blank(Y3); } while (0)

#define MakeFun(FunP, NumFree)					\
  do {								\
     SWAPOUT;							\
     reg[0] = r(0);						\
     r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree);	\
     SWAPIN;							\
  } while (0)

#define PutTuple(Dst, Arity)			\
 do {						\
   Dst = make_tuple(HTOP);			\
   pt_arity = (Arity);				\
 } while (0)

/*
 * Check that we haven't used the reductions and jump to function pointed to by
 * the I register.  If we are out of reductions, do a context switch.
 */

#define DispatchMacro()				\
  do {						\
     BeamInstr* dis_next;				\
     dis_next = (BeamInstr *) *I;			\
     CHECK_ARGS(I);				\
     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
        FCALLS--;				\
        Goto(dis_next);				\
     } else {					\
	goto context_switch;			\
     }						\
 } while (0)

#define DispatchMacroFun()			\
  do {						\
     BeamInstr* dis_next;				\
     dis_next = (BeamInstr *) *I;			\
     CHECK_ARGS(I);				\
     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
        FCALLS--;				\
        Goto(dis_next);				\
     } else {					\
	goto context_switch_fun;		\
     }						\
 } while (0)

#define DispatchMacrox()					\
  do {								\
     if (FCALLS > 0) {						\
        Eterm* dis_next;					\
        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
        dis_next = (Eterm *) *I;				\
        FCALLS--;						\
        CHECK_ARGS(I);						\
        Goto(dis_next);						\
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)		\
		&& FCALLS > neg_o_reds) {			\
        goto save_calls1;					\
     } else {							\
        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]); \
        CHECK_ARGS(I);						\
	goto context_switch;					\
     }								\
 } while (0)

#ifdef DEBUG
/*
 * To simplify breakpoint setting, put the code in one place only and jump to it.
 */
#  define Dispatch() goto do_dispatch
#  define Dispatchx() goto do_dispatchx
#  define Dispatchfun() goto do_dispatchfun
#else
# 511 "beam/beam_emu.c"
/*
 * Inline for speed.
 */
#  define Dispatch() DispatchMacro()
#  define Dispatchx() DispatchMacrox()
#  define Dispatchfun() DispatchMacroFun()
#endif
# 518 "beam/beam_emu.c"

#define Self(R) R = c_p->common.id
#define Node(R) R = erts_this_node->sysname

#define Arg(N)       I[(N)+1]
#define Next(N)                \
    I += (N) + 1;              \
    ASSERT(VALID_INSTR(*I));   \
    Goto(*I)

#define PreFetch(N, Dst) do { Dst = (BeamInstr *) *(I + N + 1); } while (0)
#define NextPF(N, Dst)         \
    I += N + 1;                \
    ASSERT(VALID_INSTR(Dst));  \
    Goto(Dst)

#define GetR(pos, tr) \
   do { \
     tr = Arg(pos); \
     switch (beam_reg_tag(tr)) { \
     case R_REG_DEF: tr = r(0); break; \
     case X_REG_DEF: tr = xb(x_reg_offset(tr)); break; \
     case Y_REG_DEF: ASSERT(y_reg_offset(tr) >= 1); tr = yb(y_reg_offset(tr)); break; \
     } \
     CHECK_TERM(tr); \
   } while (0)

#define GetArg1(N, Dst) GetR((N), Dst)

#define GetArg2(N, Dst1, Dst2)     \
   do {                            \
     GetR(N, Dst1);                \
     GetR((N)+1, Dst2);            \
   } while (0)

#define PutList(H, T, Dst, Store)  \
  do {                             \
   HTOP[0] = (H); HTOP[1] = (T);   \
   Store(make_list(HTOP), Dst);    \
   HTOP += 2;                      \
  } while (0)

#define Move(Src, Dst, Store)      \
   do {                            \
       Eterm term = (Src);         \
       Store(term, Dst);           \
   } while (0)

#define Move2(S1, D1, S2, D2) D1 = (S1); D2 = (S2)
#define Move3(S1, D1, S2, D2, S3, D3) D1 = (S1); D2 = (S2); D3 = (S3)

#define MoveGenDest(src, dstp) \
   if ((dstp) == NULL) { r(0) = (src); } else { *(dstp) = src; }

#define MoveReturn(Src, Dest)       \
    (Dest) = (Src);                 \
    I = c_p->cp;                    \
    ASSERT(VALID_INSTR(*c_p->cp));  \
    c_p->cp = 0;                    \
    CHECK_TERM(r(0));               \
    Goto(*I)

#define DeallocateReturn(Deallocate)       \
  do {                                     \
    int words_to_pop = (Deallocate);       \
    SET_I((BeamInstr *) cp_val(*E));                     \
    E = ADD_BYTE_OFFSET(E, words_to_pop);  \
    CHECK_TERM(r(0));                      \
    Goto(*I);                              \
  } while (0)

#define MoveDeallocateReturn(Src, Dest, Deallocate)  \
    (Dest) = (Src);                                  \
    DeallocateReturn(Deallocate)

#define MoveCall(Src, Dest, CallDest, Size)	\
    (Dest) = (Src);				\
    SET_CP(c_p, I+Size+1);			\
    SET_I((BeamInstr *) CallDest);			\
    Dispatch();

#define MoveCallLast(Src, Dest, CallDest, Deallocate)	\
    (Dest) = (Src);					\
    RESTORE_CP(E);					\
    E = ADD_BYTE_OFFSET(E, (Deallocate));		\
    SET_I((BeamInstr *) CallDest);				\
    Dispatch();

#define MoveCallOnly(Src, Dest, CallDest)	\
    (Dest) = (Src);				\
    SET_I((BeamInstr *) CallDest);			\
    Dispatch();

#define MoveJump(Src)				\
     r(0) = (Src);				\
     SET_I((BeamInstr *) Arg(0));		\
     Goto(*I);

#define GetList(Src, H, T) do {			\
   Eterm* tmp_ptr = list_val(Src);		\
   H = CAR(tmp_ptr);				\
   T = CDR(tmp_ptr); } while (0)

#define GetTupleElement(Src, Element, Dest)				\
  do {									\
    tmp_arg1 = (Eterm) COMPRESS_POINTER(((unsigned char *) tuple_val(Src)) + 	\
				(Element));				\
    (Dest) = (*(Eterm *) EXPAND_POINTER(tmp_arg1));			\
  } while (0)

#define ExtractNextElement(Dest)					  \
    tmp_arg1 += sizeof(Eterm);						  \
    (Dest) = (* (Eterm *) (((unsigned char *) EXPAND_POINTER(tmp_arg1))))

#define ExtractNextElement2(Dest)				\
  do {								\
    Eterm* ene_dstp = &(Dest);					\
    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
    tmp_arg1 += sizeof(Eterm) + sizeof(Eterm);			\
  } while (0)

#define ExtractNextElement3(Dest)		\
  do {						\
    Eterm* ene_dstp = &(Dest);			\
    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
    tmp_arg1 += 3*sizeof(Eterm);		\
  } while (0)

#define ExtractNextElement4(Dest)		\
  do {						\
    Eterm* ene_dstp = &(Dest);			\
    ene_dstp[0] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[1];	\
    ene_dstp[1] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[2];	\
    ene_dstp[2] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[3];	\
    ene_dstp[3] = ((Eterm *) EXPAND_POINTER(tmp_arg1))[4];	\
    tmp_arg1 += 4*sizeof(Eterm);		\
  } while (0)

#define ExtractElement(Element, Dest)		\
  do {						\
     tmp_arg1 += (Element);			\
     (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1));		\
  } while (0)

#define EqualImmed(X, Y, Action) if (X != Y) { Action; }
#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; }
#define EqualExact(X, Y, Action) if (!EQ(X,Y)) { Action; }
#define IsLessThan(X, Y, Action) if (CMP_GE(X, Y)) { Action; }
#define IsGreaterEqual(X, Y, Action) if (CMP_LT(X, Y)) { Action; }

#define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; }

#define IsInteger(Src, Fail) if (is_not_integer(Src)) { Fail; }

#define IsNumber(X, Fail) if (is_not_integer(X) && is_not_float(X)) { Fail; }

#define IsAtom(Src, Fail) if (is_not_atom(Src)) { Fail; }

#define IsIntegerAllocate(Src, Need, Alive, Fail)  \
    if (is_not_integer(Src)) { Fail; }             \
    A(Need, Alive)

#define IsNil(Src, Fail) if (is_not_nil(Src)) { Fail; }

#define IsList(Src, Fail) if (is_not_list(Src) && is_not_nil(Src)) { Fail; }

#define IsNonemptyList(Src, Fail) if (is_not_list(Src)) { Fail; }

#define IsNonemptyListAllocate(Src, Need, Alive, Fail)  \
    if (is_not_list(Src)) { Fail; }                     \
    A(Need, Alive)

#define IsNonemptyListTestHeap(Src, Need, Alive, Fail)  \
    if (is_not_list(Src)) { Fail; }                     \
    TestHeap(Need, Alive)

#define IsTuple(X, Action) if (is_not_tuple(X)) Action

#define IsArity(Pointer, Arity, Fail)					  \
    if (*(Eterm *)							  \
	EXPAND_POINTER(tmp_arg1 = (Eterm) 				  \
		       COMPRESS_POINTER(tuple_val(Pointer))) != (Arity))  \
    { 									  \
        Fail; 								  \
    }

#define IsMap(Src, Fail) if (!is_map(Src)) { Fail; }

#define GetMapElement(Src, Key, Dst, Fail)	\
  do {						\
     Eterm _res = get_map_element(Src, Key);	\
     if (is_non_value(_res)) {			\
        Fail;					\
     }						\
     Dst = _res;				\
  } while (0)

#define GetMapElementHash(Src, Key, Hx, Dst, Fail)	\
  do {							\
     Eterm _res = get_map_element_hash(Src, Key, Hx);	\
     if (is_non_value(_res)) {				\
        Fail;						\
     }							\
     Dst = _res;					\
  } while (0)

#define IsFunction(X, Action)			\
  do {						\
     if ( !(is_any_fun(X)) ) {			\
          Action;				\
     }						\
  } while (0)

#define IsFunction2(F, A, Action)			\
  do {							\
     if (erl_is_function(c_p, F, A) != am_true ) {	\
          Action;					\
     }							\
  } while (0)

#define IsTupleOfArity(Src, Arity, Fail)				      \
  do {									      \
    if (is_not_tuple(Src) || 						      \
	*(Eterm *)							      \
	EXPAND_POINTER(tmp_arg1 = 					      \
		       (Eterm) COMPRESS_POINTER(tuple_val(Src))) != Arity) { \
        Fail;								      \
    }									      \
  } while (0)

#define IsBoolean(X, Fail) if ((X) != am_true && (X) != am_false) { Fail; }

#define IsBinary(Src, Fail) \
 if (is_not_binary(Src) || binary_bitsize(Src) != 0) { Fail; }

#define IsBitstring(Src, Fail) \
  if (is_not_binary(Src)) { Fail; }

#if defined(ARCH_64) && !HALFWORD_HEAP
#define BsSafeMul(A, B, Fail, Target)		\
   do { Uint64 _res = (A) * (B);		\
      if (_res / B != A) { Fail; }		\
      Target = _res;				\
   } while (0)
#else
# 766 "beam/beam_emu.c"
#define BsSafeMul(A, B, Fail, Target)			\
   do { Uint64 _res = (Uint64)(A) * (Uint64)(B);	\
      if ((_res >> (8*sizeof(Uint))) != 0) { Fail; }	\
      Target = _res;					\
   } while (0)
#endif
# 772 "beam/beam_emu.c"

#define BsGetFieldSize(Bits, Unit, Fail, Target)	\
   do {							\
      Sint _signed_size; Uint _uint_size;		\
      if (is_small(Bits)) {				\
        _signed_size = signed_val(Bits);		\
         if (_signed_size < 0) { Fail; }		\
         _uint_size = (Uint) _signed_size;		\
      } else {						\
         if (!term_to_Uint(Bits, &temp_bits)) { Fail; }	\
         _uint_size = temp_bits;			\
      }							\
      BsSafeMul(_uint_size, Unit, Fail, Target);	\
   } while (0)

#define BsGetUncheckedFieldSize(Bits, Unit, Fail, Target)	\
   do {								\
      Sint _signed_size; Uint _uint_size;			\
      if (is_small(Bits)) {					\
        _signed_size = signed_val(Bits);			\
         if (_signed_size < 0) { Fail; }			\
         _uint_size = (Uint) _signed_size;			\
      } else {							\
         if (!term_to_Uint(Bits, &temp_bits)) { Fail; }		\
         _uint_size = (Uint) temp_bits;				\
      }								\
      Target = _uint_size * Unit;				\
   } while (0)

#define BsGetFloat2(Ms, Live, Sz, Flags, Dst, Store, Fail)		\
 do {									\
   ErlBinMatchBuffer *_mb;						\
   Eterm _result; Sint _size;						\
   if (!is_small(Sz) || (_size = unsigned_val(Sz)) > 64) { Fail; }	\
   _size *= ((Flags) >> 3);						\
   TestHeap(FLOAT_SIZE_OBJECT, Live);					\
   _mb = ms_matchbuffer(Ms);						\
   LIGHT_SWAPOUT;							\
   _result = erts_bs_get_float_2(c_p, _size, (Flags), _mb);		\
   LIGHT_SWAPIN;							\
   HEAP_SPACE_VERIFIED(0);                                          \
   if (is_non_value(_result)) { Fail; }					\
   else { Store(_result, Dst); }					\
 } while (0)

#define BsGetBinaryImm_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
  do {								\
    ErlBinMatchBuffer *_mb;					\
    Eterm _result;						\
    TestHeap(heap_bin_size(ERL_ONHEAP_BIN_LIMIT), Live);	\
    _mb = ms_matchbuffer(Ms);					\
    LIGHT_SWAPOUT;						\
    _result = erts_bs_get_binary_2(c_p, (Sz), (Flags), _mb);	\
    LIGHT_SWAPIN;						\
    HEAP_SPACE_VERIFIED(0);                                 \
    if (is_non_value(_result)) { Fail; }			\
    else { Store(_result, Dst); }				\
  } while (0)

#define BsGetBinary_2(Ms, Live, Sz, Flags, Dst, Store, Fail)	\
  do {								\
    ErlBinMatchBuffer *_mb;					\
    Eterm _result; Uint _size;					\
    BsGetFieldSize(Sz, ((Flags) >> 3), Fail, _size);		\
    TestHeap(ERL_SUB_BIN_SIZE, Live);				\
    _mb = ms_matchbuffer(Ms);					\
    LIGHT_SWAPOUT;						\
    _result = erts_bs_get_binary_2(c_p, _size, (Flags), _mb);	\
    LIGHT_SWAPIN;						\
    HEAP_SPACE_VERIFIED(0);                                 \
    if (is_non_value(_result)) { Fail; }			\
    else { Store(_result, Dst); }				\
  } while (0)

#define BsGetBinaryAll_2(Ms, Live, Unit, Dst, Store, Fail)	\
  do {								\
    ErlBinMatchBuffer *_mb;					\
    Eterm _result;						\
    TestHeap(ERL_SUB_BIN_SIZE, Live);				\
    _mb = ms_matchbuffer(Ms);					\
    if (((_mb->size - _mb->offset) % Unit) == 0) {		\
      LIGHT_SWAPOUT;						\
      _result = erts_bs_get_binary_all_2(c_p, _mb);		\
      LIGHT_SWAPIN;						\
      HEAP_SPACE_VERIFIED(0);                               \
      ASSERT(is_value(_result));				\
      Store(_result, Dst);					\
    } else {                                                    \
	HEAP_SPACE_VERIFIED(0);                             \
	Fail; }						        \
 } while (0)

#define BsSkipBits2(Ms, Bits, Unit, Fail)			\
 do {								\
   ErlBinMatchBuffer *_mb;					\
   size_t new_offset;						\
   Uint _size;							\
    _mb = ms_matchbuffer(Ms);					\
   BsGetFieldSize(Bits, Unit, Fail, _size);			\
   new_offset = _mb->offset + _size;				\
   if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
   else { Fail; }						\
 } while (0)

#define BsSkipBitsAll2(Ms, Unit, Fail)		\
 do {						\
    ErlBinMatchBuffer *_mb;			\
   _mb = ms_matchbuffer(Ms);			\
   if (((_mb->size - _mb->offset) % Unit) == 0) {_mb->offset = _mb->size; } \
   else { Fail; }					\
 } while (0)

#define BsSkipBitsImm2(Ms, Bits, Fail)				\
 do {								\
   ErlBinMatchBuffer *_mb;					\
   size_t new_offset;						\
   _mb = ms_matchbuffer(Ms);					\
   new_offset = _mb->offset + (Bits);				\
   if (new_offset <= _mb->size) { _mb->offset = new_offset; }	\
   else { Fail; }						\
 } while (0)

#define NewBsPutIntegerImm(Sz, Flags, Src)					\
 do {									\
    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), (Sz), (Flags)))) { goto badarg; }	\
 } while (0)

#define NewBsPutInteger(Sz, Flags, Src)						\
 do {										\
    Sint _size;									\
    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
    if (!erts_new_bs_put_integer(ERL_BITS_ARGS_3((Src), _size, (Flags))))	\
       { goto badarg; }								\
 } while (0)

#define NewBsPutFloatImm(Sz, Flags, Src)					\
 do {									\
    if (!erts_new_bs_put_float(c_p, (Src), (Sz), (Flags))) { goto badarg; }	\
 } while (0)

#define NewBsPutFloat(Sz, Flags, Src)						\
 do {										\
    Sint _size;									\
    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);		\
    if (!erts_new_bs_put_float(c_p, (Src), _size, (Flags))) { goto badarg; }	\
 } while (0)

#define NewBsPutBinary(Sz, Flags, Src)							\
 do {											\
    Sint _size;										\
    BsGetUncheckedFieldSize(Sz, ((Flags) >> 3), goto badarg, _size);			\
    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), _size))) { goto badarg; }	\
 } while (0)

#define NewBsPutBinaryImm(Sz, Src)				        \
 do {							        \
    if (!erts_new_bs_put_binary(ERL_BITS_ARGS_2((Src), (Sz)))) { goto badarg; }	\
 } while (0)

#define NewBsPutBinaryAll(Src, Unit)							\
 do {											\
    if (!erts_new_bs_put_binary_all(ERL_BITS_ARGS_2((Src), (Unit)))) { goto badarg; }	\
 } while (0)


#define IsPort(Src, Fail) if (is_not_port(Src)) { Fail; }
#define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; }
#define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; }

/*
 * process_main() is already huge, so we want to avoid inlining
 * into it. Especially functions that are seldom used.
 */
#ifdef __GNUC__
#  define NOINLINE __attribute__((__noinline__))
#else
# 948 "beam/beam_emu.c"
#  define NOINLINE
#endif
# 950 "beam/beam_emu.c"


/*
 * The following functions are called directly by process_main().
 * Don't inline them.
 */
static BifFunction translate_gc_bif(void* gcf) NOINLINE;
static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
			       Eterm* reg, BifFunction bf) NOINLINE;
static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
				     Eterm* reg, Eterm func) NOINLINE;
static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
static BeamInstr* apply(Process* p, Eterm module, Eterm function,
			Eterm args, Eterm* reg) NOINLINE;
static BeamInstr* call_fun(Process* p, int arity,
			   Eterm* reg, Eterm args) NOINLINE;
static BeamInstr* apply_fun(Process* p, Eterm fun,
			    Eterm args, Eterm* reg) NOINLINE;
static Eterm new_fun(Process* p, Eterm* reg,
		     ErlFunEntry* fe, int num_free) NOINLINE;
static Eterm new_map(Process* p, Eterm* reg, BeamInstr* I) NOINLINE;
static Eterm update_map_assoc(Process* p, Eterm* reg,
			      Eterm map, BeamInstr* I) NOINLINE;
static Eterm update_map_exact(Process* p, Eterm* reg,
			      Eterm map, BeamInstr* I) NOINLINE;
static Eterm get_map_element(Eterm map, Eterm key);
static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx);

/*
 * Functions not directly called by process_main(). OK to inline.
 */
static BeamInstr* next_catch(Process* c_p, Eterm *reg);
static void terminate_proc(Process* c_p, Eterm Value);
static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
			     BifFunction bf, Eterm args);
static struct StackTrace * get_trace_from_exc(Eterm exc);
static Eterm make_arglist(Process* c_p, Eterm* reg, int a);

void
init_emulator(void)
{
    process_main();
}

/*
 * On certain platforms, make sure that the main variables really are placed
 * in registers.
 */

#if defined(__GNUC__) && defined(sparc) && !defined(DEBUG)
#  define REG_x0 asm("%l0")
#  define REG_xregs asm("%l1")
#  define REG_htop asm("%l2")
#  define REG_stop asm("%l3")
#  define REG_I asm("%l4")
#  define REG_fcalls asm("%l5")
#  define REG_tmp_arg1 asm("%l6")
#  define REG_tmp_arg2 asm("%l7")
#else
# 1010 "beam/beam_emu.c"
#  define REG_x0
#  define REG_xregs
#  define REG_htop
#  define REG_stop
#  define REG_I
#  define REG_fcalls
#  define REG_tmp_arg1
#  define REG_tmp_arg2
#endif
# 1019 "beam/beam_emu.c"

#ifdef USE_VM_PROBES
#  define USE_VM_CALL_PROBES
#endif
# 1023 "beam/beam_emu.c"

#ifdef USE_VM_CALL_PROBES

#define DTRACE_LOCAL_CALL(p, m, f, a)					\
    if (DTRACE_ENABLED(local_function_entry)) {				\
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);		\
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);			\
        int depth = STACK_START(p) - STACK_TOP(p);			\
        dtrace_fun_decode(p, m, f, a,					\
                          process_name, mfa);				\
        DTRACE3(local_function_entry, process_name, mfa, depth);	\
    }

#define DTRACE_GLOBAL_CALL(p, m, f, a)					\
    if (DTRACE_ENABLED(global_function_entry)) {			\
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);		\
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);			\
        int depth = STACK_START(p) - STACK_TOP(p);			\
        dtrace_fun_decode(p, m, f, a,					\
                          process_name, mfa);				\
        DTRACE3(global_function_entry, process_name, mfa, depth);	\
    }

#define DTRACE_RETURN(p, m, f, a)                               \
    if (DTRACE_ENABLED(function_return)) {                      \
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
        int depth = STACK_START(p) - STACK_TOP(p);              \
        dtrace_fun_decode(p, m, f, a,                           \
                          process_name, mfa);                   \
        DTRACE3(function_return, process_name, mfa, depth);     \
    }

#define DTRACE_BIF_ENTRY(p, m, f, a)                            \
    if (DTRACE_ENABLED(bif_entry)) {                            \
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
        dtrace_fun_decode(p, m, f, a,                           \
                          process_name, mfa);                   \
        DTRACE2(bif_entry, process_name, mfa);                  \
    }

#define DTRACE_BIF_RETURN(p, m, f, a)                           \
    if (DTRACE_ENABLED(bif_return)) {                           \
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
        dtrace_fun_decode(p, m, f, a,                           \
                          process_name, mfa);                   \
        DTRACE2(bif_return, process_name, mfa);                 \
    }

#define DTRACE_NIF_ENTRY(p, m, f, a)                            \
    if (DTRACE_ENABLED(nif_entry)) {                            \
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
        dtrace_fun_decode(p, m, f, a,                           \
                          process_name, mfa);                   \
        DTRACE2(nif_entry, process_name, mfa);                  \
    }

#define DTRACE_NIF_RETURN(p, m, f, a)                           \
    if (DTRACE_ENABLED(nif_return)) {                           \
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);     \
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);              \
        dtrace_fun_decode(p, m, f, a,                           \
                          process_name, mfa);                   \
        DTRACE2(nif_return, process_name, mfa);                 \
    }

#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e)                                                    \
    do {                                                                                       \
        if (DTRACE_ENABLED(global_function_entry)) {                                           \
            BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \
            DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]);                     \
        }                                                                                      \
    } while(0)

#define DTRACE_RETURN_FROM_PC(p)                                                        \
    do {                                                                                \
        BeamInstr* fp;                                                                  \
        if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \
            DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]);                \
        }                                                                               \
    } while(0)

#else /* USE_VM_PROBES */
# 1109 "beam/beam_emu.c"
#define DTRACE_LOCAL_CALL(p, m, f, a)        do {} while (0)
#define DTRACE_GLOBAL_CALL(p, m, f, a)       do {} while (0)
#define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0)
#define DTRACE_RETURN(p, m, f, a)            do {} while (0)
#define DTRACE_RETURN_FROM_PC(p)             do {} while (0)
#define DTRACE_BIF_ENTRY(p, m, f, a)         do {} while (0)
#define DTRACE_BIF_RETURN(p, m, f, a)        do {} while (0)
#define DTRACE_NIF_ENTRY(p, m, f, a)         do {} while (0)
#define DTRACE_NIF_RETURN(p, m, f, a)        do {} while (0)
#endif /* USE_VM_PROBES */
# 1119 "beam/beam_emu.c"

/*
 * process_main() is called twice:
 * The first call performs some initialisation, including exporting
 * the instructions' C labels to the loader.
 * The second call starts execution of BEAM code. This call never returns.
 */
void process_main(void)
{
    static int init_done = 0;
    Process* c_p = NULL;
    int reds_used;
#ifdef DEBUG
    ERTS_DECLARE_DUMMY(Eterm pid);
#endif
# 1134 "beam/beam_emu.c"

    /*
     * X register zero; also called r(0)
     */
    register Eterm x0 REG_x0 = NIL;

    /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
     * in all other cases x0 is used.
     */
    register Eterm* reg REG_xregs = NULL;

    /*
     * Top of heap (next free location); grows upwards.
     */
    register Eterm* HTOP REG_htop = NULL;

    /* Stack pointer.  Grows downwards; points
     * to last item pushed (normally a saved
     * continuation pointer).
     */
    register Eterm* E REG_stop = NULL;

    /*
     * Pointer to next threaded instruction.
     */
    register BeamInstr *I REG_I = NULL;

    /* Number of reductions left.  This function
     * returns to the scheduler when FCALLS reaches zero.
     */
    register Sint FCALLS REG_fcalls = 0;

    /*
     * Temporaries used for picking up arguments for instructions.
     */
    register Eterm tmp_arg1 REG_tmp_arg1 = NIL;
    register Eterm tmp_arg2 REG_tmp_arg2 = NIL;
#if HEAP_ON_C_STACK
    Eterm tmp_big[2];           /* Temporary buffer for small bignums if HEAP_ON_C_STACK. */
#else
# 1174 "beam/beam_emu.c"
    Eterm *tmp_big;		/* Temporary buffer for small bignums if !HEAP_ON_C_STACK. */
#endif
# 1176 "beam/beam_emu.c"

    /*
     * X registers and floating point registers are located in
     * scheduler specific data.
     */
    register FloatDef *freg;

    /*
     * For keeping the negative old value of 'reds' when call saving is active.
     */
    int neg_o_reds = 0;

    Eterm (*arith_func)(Process* p, Eterm* reg, Uint live);

#ifdef ERTS_OPCODE_COUNTER_SUPPORT
    static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
#else
# 1193 "beam/beam_emu.c"
#ifndef NO_JUMP_TABLE
    static void* opcodes[] = { DEFINE_OPCODES };
#else
# 1196 "beam/beam_emu.c"
    int Go;
#endif
# 1198 "beam/beam_emu.c"
#endif
# 1199 "beam/beam_emu.c"

    Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */

    Eterm pt_arity;		/* Used by do_put_tuple */

    Uint64 start_time = 0;          /* Monitor long schedule */
    BeamInstr* start_time_i = NULL;

    ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */


    /*
     * Note: In this function, we attempt to place rarely executed code towards
     * the end of the function, in the hope that the cache hit rate will be better.
     * The initialization code is only run once, so it is at the very end.
     *
     * Note: c_p->arity must be set to reflect the number of useful terms in
     * c_p->arg_reg before calling the scheduler.
     */
    if (!init_done) {
       /* This should only be reached during the init phase when only the main
        * process is running. I.e. there is no race for init_done.
        */
	init_done = 1;
	goto init_emulator;
    }

    c_p = NULL;
    reds_used = 0;

    goto do_schedule1;

 do_schedule:
    reds_used = REDS_IN(c_p) - FCALLS;
 do_schedule1:

    if (start_time != 0) {
        Sint64 diff = erts_timestamp_millis() - start_time;
	if (diff > 0 && (Uint) diff >  erts_system_monitor_long_schedule
#ifdef ERTS_DIRTY_SCHEDULERS
	    && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data)
#endif
# 1241 "beam/beam_emu.c"
	    ) {
	    BeamInstr *inptr = find_function_from_pc(start_time_i);
	    BeamInstr *outptr = find_function_from_pc(c_p->i);
	    monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff);
	}
    }

    PROCESS_MAIN_CHK_LOCKS(c_p);
    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
#if HALFWORD_HEAP
    ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0);
#endif
# 1253 "beam/beam_emu.c"
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    c_p = schedule(c_p, reds_used);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    start_time = 0;
#ifdef DEBUG
    pid = c_p->common.id; /* Save for debugging purpouses */
#endif
# 1260 "beam/beam_emu.c"
    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    PROCESS_MAIN_CHK_LOCKS(c_p);

    if (erts_system_monitor_long_schedule != 0) {
	start_time = erts_timestamp_millis();
	start_time_i = c_p->i;
    }

    reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
    freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
#if !HEAP_ON_C_STACK
    tmp_big = ERTS_PROC_GET_SCHDATA(c_p)->beam_emu_tmp_heap;
#endif
# 1273 "beam/beam_emu.c"
    ERL_BITS_RELOAD_STATEP(c_p);
    {
	int reds;
	Eterm* argp;
	BeamInstr *next;
	int i;

	argp = c_p->arg_reg;
	for (i = c_p->arity - 1; i > 0; i--) {
	    reg[i] = argp[i];
	    CHECK_TERM(reg[i]);
	}

	/*
	 * We put the original reduction count in the process structure, to reduce
	 * the code size (referencing a field in a struct through a pointer stored
	 * in a register gives smaller code than referencing a global variable).
	 */

	SET_I(c_p->i);

	reds = c_p->fcalls;
	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)
	    && (ERTS_TRACE_FLAGS(c_p) & F_SENSITIVE) == 0) {
	    neg_o_reds = -reds;
	    FCALLS = REDS_IN(c_p) = 0;
	} else {
	    neg_o_reds = 0;
	    FCALLS = REDS_IN(c_p) = reds;
	}

	next = (BeamInstr *) *I;
	r(0) = c_p->arg_reg[0];
#ifdef HARDDEBUG
	if (c_p->arity > 0) {
	    CHECK_TERM(r(0));
	}
#endif
# 1311 "beam/beam_emu.c"
	SWAPIN;
	ASSERT(VALID_INSTR(next));

#ifdef USE_VM_PROBES
        if (DTRACE_ENABLED(process_scheduled)) {
            DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
            DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
            dtrace_proc_str(c_p, process_buf);

            if (ERTS_PROC_IS_EXITING(c_p)) {
                strcpy(fun_buf, "<exiting>");
            } else {
                BeamInstr *fptr = find_function_from_pc(c_p->i);
                if (fptr) {
                    dtrace_fun_decode(c_p, (Eterm)fptr[0],
                                      (Eterm)fptr[1], (Uint)fptr[2],
                                      NULL, fun_buf);
                } else {
                    erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
                                  "<unknown/%p>", next);
                }
            }

            DTRACE2(process_scheduled, process_buf, fun_buf);
        }
#endif
# 1337 "beam/beam_emu.c"
	Goto(next);
    }

#if defined(DEBUG) || defined(NO_JUMP_TABLE)
 emulator_loop:
#endif
# 1343 "beam/beam_emu.c"

#ifdef NO_JUMP_TABLE
    switch (Go) {
#endif
# 1347 "beam/beam_emu.c"
#if 0 /* expanded by -frewrite-includes */
#include "beam_hot.h"
#endif /* expanded by -frewrite-includes */
# 1347 "beam/beam_emu.c"
# 1 "armv6-portbld-freebsd11.0/opt/smp/beam_hot.h" 1
/*
 *  Warning: Do not edit this file.
 *  Auto-generated by 'beam_makeops'.
 */

OpCase(allocate_heap_tIt):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    AllocateHeap(tb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), tb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(allocate_heap_zero_tIt):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    AllocateHeapZero(tb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), tb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(allocate_init_tIy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    AllocateInit(tb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(allocate_tt):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Allocate(tb(tmp_packed1&BEAM_LOOSE_MASK), tb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(allocate_zero_tt):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    AllocateZero(tb(tmp_packed1&BEAM_LOOSE_MASK), tb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(deallocate_return_Q):
    { 
    DeallocateReturn(Arg(0));
    }

OpCase(extract_next_element2_x):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    ExtractNextElement2(xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(extract_next_element2_y):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    ExtractNextElement2(yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(extract_next_element3_x):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    ExtractNextElement3(xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(extract_next_element3_y):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    ExtractNextElement3(yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(extract_next_element_x):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    ExtractNextElement(xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(extract_next_element_y):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    ExtractNextElement(yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(get_list_rrx):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    GetList(r(0), r(0), xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(get_list_rry):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    GetList(r(0), r(0), yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(get_list_rxr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    GetList(r(0), xb(Arg(0)), r(0));
    NextPF(1, next);
    }

OpCase(get_list_rxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(r(0), xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_rxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(r(0), xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_ryr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    GetList(r(0), yb(Arg(0)), r(0));
    NextPF(1, next);
    }

OpCase(get_list_ryx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(r(0), yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_ryy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(r(0), yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_xrx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_LOOSE_MASK), r(0), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_xry):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_LOOSE_MASK), r(0), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_xxr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0));
    NextPF(1, next);
    }

OpCase(get_list_xxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_xxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_xyr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0));
    NextPF(1, next);
    }

OpCase(get_list_xyx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_xyy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(xb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_yrx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_LOOSE_MASK), r(0), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_yry):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_LOOSE_MASK), r(0), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(get_list_yxr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0));
    NextPF(1, next);
    }

OpCase(get_list_yxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_yxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_yyr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0));
    NextPF(1, next);
    }

OpCase(get_list_yyx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(get_list_yyy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    GetList(yb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(i_fetch_cr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    FetchArgs(Arg(0), r(0));
    NextPF(1, next);
    }

OpCase(i_fetch_cx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    FetchArgs(Arg(0), xb(Arg(1)));
    NextPF(2, next);
    }

OpCase(i_fetch_cy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    FetchArgs(Arg(0), yb(Arg(1)));
    NextPF(2, next);
    }

OpCase(i_fetch_rc):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    FetchArgs(r(0), Arg(0));
    NextPF(1, next);
    }

OpCase(i_fetch_rx):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    FetchArgs(r(0), xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(i_fetch_ry):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    FetchArgs(r(0), yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(i_fetch_xc):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    FetchArgs(xb(Arg(0)), Arg(1));
    NextPF(2, next);
    }

OpCase(i_fetch_xr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    FetchArgs(xb(Arg(0)), r(0));
    NextPF(1, next);
    }

OpCase(i_fetch_xx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    FetchArgs(xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(i_fetch_xy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    FetchArgs(xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(i_fetch_yc):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    FetchArgs(yb(Arg(0)), Arg(1));
    NextPF(2, next);
    }

OpCase(i_fetch_yr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    FetchArgs(yb(Arg(0)), r(0));
    NextPF(1, next);
    }

OpCase(i_fetch_yx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    FetchArgs(yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(i_fetch_yy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    FetchArgs(yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(i_get_map_element_frxr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    GetMapElement(r(0), xb(Arg(1)), r(0), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_get_map_element_frxx):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetMapElement(r(0), xb(Arg(1)), xb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_get_map_element_frxy):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetMapElement(r(0), xb(Arg(1)), yb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_get_map_element_fxxr):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetMapElement(xb(Arg(1)), xb(Arg(2)), r(0), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_get_map_element_fxxx):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElement(xb(Arg(1)), xb(Arg(2)), xb(Arg(3)), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_fxxy):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElement(xb(Arg(1)), xb(Arg(2)), yb(Arg(3)), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_fyxr):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetMapElement(yb(Arg(1)), xb(Arg(2)), r(0), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_get_map_element_fyxx):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElement(yb(Arg(1)), xb(Arg(2)), xb(Arg(3)), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_fyxy):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElement(yb(Arg(1)), xb(Arg(2)), yb(Arg(3)), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_hash_frcIr):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetMapElementHash(r(0), Arg(1), Arg(2), r(0), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_get_map_element_hash_frcIx):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElementHash(r(0), Arg(1), Arg(2), xb(Arg(3)), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_hash_frcIy):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElementHash(r(0), Arg(1), Arg(2), yb(Arg(3)), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_hash_fxcIr):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElementHash(xb(Arg(1)), Arg(2), Arg(3), r(0), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_hash_fxcIx):
    { 
    BeamInstr* next;
    PreFetch(5, next);
    GetMapElementHash(xb(Arg(1)), Arg(2), Arg(3), xb(Arg(4)), ClauseFail());
    NextPF(5, next);
    }

OpCase(i_get_map_element_hash_fxcIy):
    { 
    BeamInstr* next;
    PreFetch(5, next);
    GetMapElementHash(xb(Arg(1)), Arg(2), Arg(3), yb(Arg(4)), ClauseFail());
    NextPF(5, next);
    }

OpCase(i_get_map_element_hash_fycIr):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    GetMapElementHash(yb(Arg(1)), Arg(2), Arg(3), r(0), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_get_map_element_hash_fycIx):
    { 
    BeamInstr* next;
    PreFetch(5, next);
    GetMapElementHash(yb(Arg(1)), Arg(2), Arg(3), xb(Arg(4)), ClauseFail());
    NextPF(5, next);
    }

OpCase(i_get_map_element_hash_fycIy):
    { 
    BeamInstr* next;
    PreFetch(5, next);
    GetMapElementHash(yb(Arg(1)), Arg(2), Arg(3), yb(Arg(4)), ClauseFail());
    NextPF(5, next);
    }

OpCase(i_get_tuple_element_rPx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    GetTupleElement(r(0), Arg(0), xb(Arg(1)));
    NextPF(2, next);
    }

OpCase(i_get_tuple_element_xPr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    GetTupleElement(xb(Arg(0)), Arg(1), r(0));
    NextPF(2, next);
    }

OpCase(i_get_tuple_element_xPx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    GetTupleElement(xb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(i_get_tuple_element_yPr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    GetTupleElement(yb(Arg(0)), Arg(1), r(0));
    NextPF(2, next);
    }

OpCase(i_get_tuple_element_yPx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    GetTupleElement(yb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(i_is_eq_exact_immed_frc):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    EqualImmed(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_eq_exact_immed_fxc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    EqualImmed(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_eq_exact_immed_fyc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    EqualImmed(yb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_eq_exact_spec_frx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    EqualExact(r(0), xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_eq_exact_spec_fry):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    EqualExact(r(0), yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_eq_exact_spec_fxx):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    EqualExact(xb(Arg(1)), xb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_eq_exact_spec_fxy):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    EqualExact(xb(Arg(1)), yb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_ge_spec_fcx):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsGreaterEqual(Arg(1), xb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_ge_spec_fxc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsGreaterEqual(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_ge_spec_fxx):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsGreaterEqual(xb(Arg(1)), xb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_lt_spec_fcr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsLessThan(Arg(1), r(0), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_lt_spec_fcx):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsLessThan(Arg(1), xb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_lt_spec_frc):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsLessThan(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_lt_spec_frx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsLessThan(r(0), xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_lt_spec_fxc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsLessThan(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_lt_spec_fxr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsLessThan(xb(Arg(1)), r(0), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_lt_spec_fxx):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsLessThan(xb(Arg(1)), xb(Arg(2)), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_ne_exact_immed_frc):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    NotEqualImmed(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_is_ne_exact_immed_fxc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    NotEqualImmed(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_ne_exact_immed_fyc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    NotEqualImmed(yb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_put_tuple_rI):
    { 
    PutTuple(r(0), Arg(0));
    I += 1 + 1;
    goto do_put_tuple;
    }

OpCase(i_put_tuple_xI):
    { 
    PutTuple(xb(Arg(0)), Arg(1));
    I += 2 + 1;
    goto do_put_tuple;
    }

OpCase(i_put_tuple_yI):
    { 
    PutTuple(yb(Arg(0)), Arg(1));
    I += 2 + 1;
    goto do_put_tuple;
    }

OpCase(init2_yy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Init2(yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    NextPF(1, next);
    }

OpCase(init3_yyy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Init3(yb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(is_atom_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsAtom(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_atom_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsAtom(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_binary_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsBinary(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_binary_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsBinary(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_bitstring_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsBitstring(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_bitstring_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsBitstring(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_float_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsFloat(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_float_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsFloat(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_function2_fss):
    { Eterm targ1; Eterm targ2; 
    BeamInstr* next;
    PreFetch(3, next);
    GetR(1, targ1);
    GetR(2, targ2);
    IsFunction2(targ1, targ2, ClauseFail());
    NextPF(3, next);
    }

OpCase(is_function_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsFunction(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_function_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsFunction(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_function_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsFunction(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_integer_allocate_frII):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsIntegerAllocate(r(0), Arg(1), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(is_integer_allocate_fxII):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    IsIntegerAllocate(xb(Arg(1)), Arg(2), Arg(3), ClauseFail());
    NextPF(4, next);
    }

OpCase(is_integer_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsInteger(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_integer_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsInteger(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_integer_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsInteger(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_list_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsList(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_list_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsList(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_map_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsMap(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_map_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsMap(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_map_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsMap(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_nil_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsNil(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_nil_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsNil(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_nil_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsNil(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_non_empty_list_test_heap_frIt):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsNonemptyListTestHeap(r(0), Arg(1), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(is_nonempty_list_allocate_frIt):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsNonemptyListAllocate(r(0), Arg(1), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(is_nonempty_list_allocate_fxIt):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(3, next);
    tmp_packed1 = Arg(1);
    IsNonemptyListAllocate(xb(tmp_packed1&BEAM_LOOSE_MASK), Arg(2), tb((tmp_packed1>>BEAM_LOOSE_SHIFT)), ClauseFail());
    NextPF(3, next);
    }

OpCase(is_nonempty_list_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsNonemptyList(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_nonempty_list_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsNonemptyList(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_nonempty_list_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsNonemptyList(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_pid_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsPid(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_pid_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsPid(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_port_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsPort(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_port_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsPort(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_reference_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsRef(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_reference_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsRef(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_tuple_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsTuple(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_tuple_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsTuple(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_tuple_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsTuple(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_tuple_of_arity_frA):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsTupleOfArity(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_tuple_of_arity_fxA):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsTupleOfArity(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(is_tuple_of_arity_fyA):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsTupleOfArity(yb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(move2_rxyx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move2(r(0), xb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(move2_ryxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move2(r(0), yb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(move2_xrxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move2(xb(tmp_packed1&BEAM_TIGHT_MASK), r(0), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(move2_xrxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move2(xb(tmp_packed1&BEAM_TIGHT_MASK), r(0), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(move2_xxxx):
    { BeamInstr tmp_packed1;BeamInstr tmp_packed2;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    tmp_packed2 = Arg(1);
    Move2(xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), xb(tmp_packed2&BEAM_LOOSE_MASK), xb((tmp_packed2>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(move2_xyxy):
    { BeamInstr tmp_packed1;BeamInstr tmp_packed2;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    tmp_packed2 = Arg(1);
    Move2(xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), xb(tmp_packed2&BEAM_LOOSE_MASK), yb((tmp_packed2>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(move2_yrxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move2(yb(tmp_packed1&BEAM_TIGHT_MASK), r(0), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), yb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(move2_yxrx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move2(yb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), r(0), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK));
    NextPF(1, next);
    }

OpCase(move2_yxyx):
    { BeamInstr tmp_packed1;BeamInstr tmp_packed2;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    tmp_packed2 = Arg(1);
    Move2(yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), yb(tmp_packed2&BEAM_LOOSE_MASK), xb((tmp_packed2>>BEAM_LOOSE_SHIFT)));
    NextPF(2, next);
    }

OpCase(move3_xxxxxx):
    { 
    BeamInstr* next;
    PreFetch(6, next);
    Move3(xb(Arg(0)), xb(Arg(1)), xb(Arg(2)), xb(Arg(3)), xb(Arg(4)), xb(Arg(5)));
    NextPF(6, next);
    }

OpCase(move3_xyxyxy):
    { 
    BeamInstr* next;
    PreFetch(6, next);
    Move3(xb(Arg(0)), yb(Arg(1)), xb(Arg(2)), yb(Arg(3)), xb(Arg(4)), yb(Arg(5)));
    NextPF(6, next);
    }

OpCase(move3_yxyxyx):
    { 
    BeamInstr* next;
    PreFetch(6, next);
    Move3(yb(Arg(0)), xb(Arg(1)), yb(Arg(2)), xb(Arg(3)), yb(Arg(4)), xb(Arg(5)));
    NextPF(6, next);
    }

OpCase(move_call_last_xrfQ):
    { BeamInstr tmp_packed1;
    tmp_packed1 = Arg(0);
    MoveCallLast(xb(tmp_packed1&BEAM_LOOSE_MASK), r(0), Arg(1), Qb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    }

OpCase(move_call_last_yrfQ):
    { BeamInstr tmp_packed1;
    tmp_packed1 = Arg(0);
    MoveCallLast(yb(tmp_packed1&BEAM_LOOSE_MASK), r(0), Arg(1), Qb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    }

OpCase(move_call_only_xrf):
    { 
    MoveCallOnly(xb(Arg(0)), r(0), Arg(1));
    }

OpCase(move_call_xrf):
    { 
    MoveCall(xb(Arg(0)), r(0), Arg(1), 2);
    }

OpCase(move_call_yrf):
    { 
    MoveCall(yb(Arg(0)), r(0), Arg(1), 2);
    }

OpCase(move_cr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Move(Arg(0), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_cx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    Move(Arg(0), xb(Arg(1)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(move_deallocate_return_crQ):
    { 
    MoveDeallocateReturn(Arg(0), r(0), Arg(1));
    }

OpCase(move_deallocate_return_nrQ):
    { 
    MoveDeallocateReturn(NIL, r(0), Arg(0));
    }

OpCase(move_deallocate_return_xrQ):
    { BeamInstr tmp_packed1;
    tmp_packed1 = Arg(0);
    MoveDeallocateReturn(xb(tmp_packed1&BEAM_LOOSE_MASK), r(0), Qb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    }

OpCase(move_deallocate_return_yrQ):
    { BeamInstr tmp_packed1;
    tmp_packed1 = Arg(0);
    MoveDeallocateReturn(yb(tmp_packed1&BEAM_LOOSE_MASK), r(0), Qb((tmp_packed1>>BEAM_LOOSE_SHIFT)));
    }

OpCase(move_jump_fc):
    { 
    MoveJump(Arg(1));
    }

OpCase(move_jump_fn):
    { 
    MoveJump(NIL);
    }

OpCase(move_jump_fx):
    { 
    MoveJump(xb(Arg(1)));
    }

OpCase(move_jump_fy):
    { 
    MoveJump(yb(Arg(1)));
    }

OpCase(move_nr):
    { 
    BeamInstr* next;
    PreFetch(0, next);
    Move(NIL, r(0), StoreSimpleDest);
    NextPF(0, next);
    }

OpCase(move_nx):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Move(NIL, xb(Arg(0)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_return_cr):
    { 
    MoveReturn(Arg(0), r(0));
    }

OpCase(move_return_nr):
    { 
    MoveReturn(NIL, r(0));
    }

OpCase(move_return_xr):
    { 
    MoveReturn(xb(Arg(0)), r(0));
    }

OpCase(move_rx):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Move(r(0), xb(Arg(0)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_ry):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Move(r(0), yb(Arg(0)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_xr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Move(xb(Arg(0)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_xx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move(xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_xy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move(xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_yr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Move(yb(Arg(0)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_yx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move(yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(move_yy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    Move(yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(node_r):
    { 
    BeamInstr* next;
    PreFetch(0, next);
    Node(r(0));
    NextPF(0, next);
    }

OpCase(node_x):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Node(xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(put_list_crr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    PutList(Arg(0), r(0), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_crx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(Arg(0), r(0), xb(Arg(1)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cry):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(Arg(0), r(0), yb(Arg(1)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cxr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(Arg(0), xb(Arg(1)), r(0), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(1);
    PutList(Arg(0), xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cxy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(1);
    PutList(Arg(0), xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cyr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(Arg(0), yb(Arg(1)), r(0), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cyx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(1);
    PutList(Arg(0), yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_cyy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(1);
    PutList(Arg(0), yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_rcr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    PutList(r(0), Arg(0), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_rcx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(r(0), Arg(0), xb(Arg(1)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_rcy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(r(0), Arg(0), yb(Arg(1)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_rnx):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    PutList(r(0), NIL, xb(Arg(0)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_rxr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    PutList(r(0), xb(Arg(0)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_rxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(r(0), xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_ryx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(r(0), yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xcr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(xb(Arg(0)), Arg(1), r(0), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_xcx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_xcy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_xnx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_LOOSE_MASK), NIL, xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xrr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    PutList(xb(Arg(0)), r(0), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xrx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_LOOSE_MASK), r(0), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xxr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xyr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_xyx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(xb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_ycr):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    PutList(yb(Arg(0)), Arg(1), r(0), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_ycx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_ycy):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(2, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_LOOSE_MASK), Arg(1), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(2, next);
    }

OpCase(put_list_ynx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_LOOSE_MASK), NIL, xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_yrr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    PutList(yb(Arg(0)), r(0), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_yrx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_LOOSE_MASK), r(0), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_yxr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_LOOSE_MASK), xb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_yxx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_TIGHT_MASK), xb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_yyr):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_LOOSE_MASK), yb((tmp_packed1>>BEAM_LOOSE_SHIFT)), r(0), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(put_list_yyx):
    { BeamInstr tmp_packed1;
    BeamInstr* next;
    PreFetch(1, next);
    tmp_packed1 = Arg(0);
    PutList(yb(tmp_packed1&BEAM_TIGHT_MASK), yb((tmp_packed1>>BEAM_TIGHT_SHIFT)&BEAM_TIGHT_MASK), xb((tmp_packed1>>(2*BEAM_TIGHT_SHIFT))&BEAM_TIGHT_MASK), StoreSimpleDest);
    NextPF(1, next);
    }

OpCase(self_r):
    { 
    BeamInstr* next;
    PreFetch(0, next);
    Self(r(0));
    NextPF(0, next);
    }

OpCase(self_x):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Self(xb(Arg(0)));
    NextPF(1, next);
    }

OpCase(self_y):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Self(yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(test_arity_frA):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsArity(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(test_arity_fxA):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsArity(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(test_arity_fyA):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsArity(yb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(test_heap_1_put_list_Iy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    TestHeapPutList(Arg(0), yb(Arg(1)));
    NextPF(2, next);
    }

OpCase(test_heap_It):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    TestHeap(Arg(0), Arg(1));
    NextPF(2, next);
    }

# 1348 "beam/beam_emu.c" 2

#define STORE_ARITH_RESULT(res) StoreBifResult(2, (res));
#define ARITH_FUNC(name) erts_gc_##name

	{
	    Eterm increment_reg_val;
	    Eterm increment_val;
	    Uint live;
	    Eterm result;

	OpCase(i_increment_yIId):
	    increment_reg_val = yb(Arg(0));
	    goto do_increment;

	OpCase(i_increment_xIId):
	    increment_reg_val = xb(Arg(0));
	    goto do_increment;

	OpCase(i_increment_rIId):
	    increment_reg_val = r(0);
	    I--;

	do_increment:
	    increment_val = Arg(1);
	    if (is_small(increment_reg_val)) {
		Sint i = signed_val(increment_reg_val) + increment_val;
		ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
		if (MY_IS_SSMALL(i)) {
		    result = make_small(i);
		store_result:
		    StoreBifResult(3, result);
		}
	    }

	    live = Arg(2);
	    SWAPOUT;
	    reg[0] = r(0);
	    reg[live] = increment_reg_val;
	    reg[live+1] = make_small(increment_val);
	    result = erts_gc_mixed_plus(c_p, reg, live);
	    r(0) = reg[0];
	    SWAPIN;
	    ERTS_HOLE_CHECK(c_p);
	    if (is_value(result)) {
		goto store_result;
	    }
	    ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
	    goto find_func_info;
	}

#define DO_BIG_ARITH(Func,Arg1,Arg2)     \
    do {                                 \
        Uint live = Arg(1);              \
        SWAPOUT;                         \
        reg[0] = r(0);                   \
        reg[live] = (Arg1);              \
        reg[live+1] = (Arg2);            \
        result = (Func)(c_p, reg, live); \
        r(0) = reg[0];                   \
        SWAPIN;                          \
        ERTS_HOLE_CHECK(c_p);            \
        if (is_value(result)) {          \
            StoreBifResult(4,result);    \
        }                                \
        goto lb_Cl_error;                \
    } while(0)

 OpCase(i_plus_jIxxd):
 {
     Eterm result;

     if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
	 Sint i = signed_val(xb(Arg(2))) + signed_val(xb(Arg(3)));
	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
	 if (MY_IS_SSMALL(i)) {
	     result = make_small(i);
             StoreBifResult(4, result);
	 }
     }
     DO_BIG_ARITH(ARITH_FUNC(mixed_plus), xb(Arg(2)), xb(Arg(3)));
 }

 OpCase(i_plus_jId):
 {
     Eterm result;

     if (is_both_small(tmp_arg1, tmp_arg2)) {
	 Sint i = signed_val(tmp_arg1) + signed_val(tmp_arg2);
	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
	 if (MY_IS_SSMALL(i)) {
	     result = make_small(i);
	     STORE_ARITH_RESULT(result);
	 }
     }
     arith_func = ARITH_FUNC(mixed_plus);
     goto do_big_arith2;
 }

 OpCase(i_minus_jIxxd):
 {
     Eterm result;

     if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
	 Sint i = signed_val(xb(Arg(2))) - signed_val(xb(Arg(3)));
	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
	 if (MY_IS_SSMALL(i)) {
	     result = make_small(i);
             StoreBifResult(4, result);
	 }
     }
     DO_BIG_ARITH(ARITH_FUNC(mixed_minus), xb(Arg(2)), xb(Arg(3)));
 }

 OpCase(i_minus_jId):
 {
     Eterm result;

     if (is_both_small(tmp_arg1, tmp_arg2)) {
	 Sint i = signed_val(tmp_arg1) - signed_val(tmp_arg2);
	 ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i));
	 if (MY_IS_SSMALL(i)) {
	     result = make_small(i);
	     STORE_ARITH_RESULT(result);
	 }
     }
     arith_func = ARITH_FUNC(mixed_minus);
     goto do_big_arith2;
 }

 OpCase(i_is_lt_f):
    if (CMP_GE(tmp_arg1, tmp_arg2)) {
	ClauseFail();
    }
    Next(1);

 OpCase(i_is_ge_f):
    if (CMP_LT(tmp_arg1, tmp_arg2)) {
	ClauseFail();
    }
    Next(1);

 OpCase(i_is_eq_f):
    if (CMP_NE(tmp_arg1, tmp_arg2)) {
	ClauseFail();
    }
    Next(1);

 OpCase(i_is_ne_f):
    if (CMP_EQ(tmp_arg1, tmp_arg2)) {
	ClauseFail();
    }
    Next(1);

 OpCase(i_is_eq_exact_f):
    if (!EQ(tmp_arg1, tmp_arg2)) {
	ClauseFail();
    }
    Next(1);

    {
	Eterm is_eq_exact_lit_val;

    OpCase(i_is_eq_exact_literal_xfc):
	is_eq_exact_lit_val = xb(Arg(0));
	I++;
	goto do_is_eq_exact_literal;

    OpCase(i_is_eq_exact_literal_yfc):
	is_eq_exact_lit_val = yb(Arg(0));
	I++;
	goto do_is_eq_exact_literal;

    OpCase(i_is_eq_exact_literal_rfc):
	is_eq_exact_lit_val = r(0);

    do_is_eq_exact_literal:
	if (!eq(Arg(1), is_eq_exact_lit_val)) {
	    ClauseFail();
	}
	Next(2);
    }

    {
	Eterm is_ne_exact_lit_val;

    OpCase(i_is_ne_exact_literal_xfc):
	is_ne_exact_lit_val = xb(Arg(0));
	I++;
	goto do_is_ne_exact_literal;

    OpCase(i_is_ne_exact_literal_yfc):
	is_ne_exact_lit_val = yb(Arg(0));
	I++;
	goto do_is_ne_exact_literal;

    OpCase(i_is_ne_exact_literal_rfc):
	is_ne_exact_lit_val = r(0);

    do_is_ne_exact_literal:
	if (eq(Arg(1), is_ne_exact_lit_val)) {
	    ClauseFail();
	}
	Next(2);
    }

 OpCase(move_window3_xxxy): {
     BeamInstr *next;
     Eterm xt0, xt1, xt2;
     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(3)));
     PreFetch(4, next);
     xt0  = xb(Arg(0));
     xt1  = xb(Arg(1));
     xt2  = xb(Arg(2));
     y[0] = xt0;
     y[1] = xt1;
     y[2] = xt2;
     NextPF(4, next);
 }
 OpCase(move_window4_xxxxy): {
     BeamInstr *next;
     Eterm xt0, xt1, xt2, xt3;
     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(4)));
     PreFetch(5, next);
     xt0  = xb(Arg(0));
     xt1  = xb(Arg(1));
     xt2  = xb(Arg(2));
     xt3  = xb(Arg(3));
     y[0] = xt0;
     y[1] = xt1;
     y[2] = xt2;
     y[3] = xt3;
     NextPF(5, next);
 }
 OpCase(move_window5_xxxxxy): {
     BeamInstr *next;
     Eterm xt0, xt1, xt2, xt3, xt4;
     Eterm *y = (Eterm *)(((unsigned char *)E) + (Arg(5)));
     PreFetch(6, next);
     xt0  = xb(Arg(0));
     xt1  = xb(Arg(1));
     xt2  = xb(Arg(2));
     xt3  = xb(Arg(3));
     xt4  = xb(Arg(4));
     y[0] = xt0;
     y[1] = xt1;
     y[2] = xt2;
     y[3] = xt3;
     y[4] = xt4;
     NextPF(6, next);
 }

 OpCase(i_move_call_only_fcr): {
     r(0) = Arg(1);
 }
 /* FALL THROUGH */
 OpCase(i_call_only_f): {
     SET_I((BeamInstr *) Arg(0));
     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
     Dispatch();
 }

 OpCase(i_move_call_last_fPcr): {
     r(0) = Arg(2);
 }
 /* FALL THROUGH */
 OpCase(i_call_last_fP): {
     RESTORE_CP(E);
     E = ADD_BYTE_OFFSET(E, Arg(1));
     SET_I((BeamInstr *) Arg(0));
     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
     Dispatch();
 }

 OpCase(i_move_call_crf): {
     r(0) = Arg(0);
     I++;
 }
 /* FALL THROUGH */
 OpCase(i_call_f): {
     SET_CP(c_p, I+2);
     SET_I((BeamInstr *) Arg(0));
     DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]);
     Dispatch();
 }

 OpCase(i_move_call_ext_last_ePcr): {
     r(0) = Arg(2);
 }
 /* FALL THROUGH */
 OpCase(i_call_ext_last_eP):
    RESTORE_CP(E);
    E = ADD_BYTE_OFFSET(E, Arg(1));

    /*
     * Note: The pointer to the export entry is never NULL; if the module
     * is not loaded, it points to code which will invoke the error handler
     * (see lb_call_error_handler below).
     */
    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
    Dispatchx();

 OpCase(i_move_call_ext_cre): {
     r(0) = Arg(0);
     I++;
 }
 /* FALL THROUGH */
 OpCase(i_call_ext_e):
    SET_CP(c_p, I+2);
    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
    Dispatchx();

 OpCase(i_move_call_ext_only_ecr): {
     r(0) = Arg(1);
 }
 /* FALL THROUGH */
 OpCase(i_call_ext_only_e):
    DTRACE_GLOBAL_CALL_FROM_EXPORT(c_p, Arg(0));
    Dispatchx();

 OpCase(init_y): {
     BeamInstr *next;

     PreFetch(1, next);
     make_blank(yb(Arg(0)));
     NextPF(1, next);
 }

 OpCase(i_trim_I): {
     BeamInstr *next;
     Uint words;
     Uint cp;

     words = Arg(0);
     cp = E[0];
     PreFetch(1, next);
     E += words;
     E[0] = cp;
     NextPF(1, next);
 }

 OpCase(move_x1_c): {
	x(1) = Arg(0);
	Next(1);
    }

 OpCase(move_x2_c): {
	x(2) = Arg(0);
	Next(1);
    }

 OpCase(return): {
    SET_I(c_p->cp);
    DTRACE_RETURN_FROM_PC(c_p);
    /*
     * We must clear the CP to make sure that a stale value do not
     * create a false module dependcy preventing code upgrading.
     * It also means that we can use the CP in stack backtraces.
     */
    c_p->cp = 0;
    CHECK_TERM(r(0));
    HEAP_SPACE_VERIFIED(0);
    Goto(*I);
 }

    /*
     * Send is almost a standard call-BIF with two arguments, except for:
     *    1) It cannot be traced.
     *	  2) There is no pointer to the send_2 function stored in
     *       the instruction.
     */

 OpCase(send): {
     BeamInstr *next;
     Eterm result;

     PRE_BIF_SWAPOUT(c_p);
     c_p->fcalls = FCALLS - 1;
     reg[0] = r(0);
     result = erl_send(c_p, r(0), x(1));
     PreFetch(0, next);
     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);
     if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
	 result = erts_gc_after_bif_call(c_p, result, reg, 2);
	 r(0) = reg[0];
	 E = c_p->stop;
     }
     HTOP = HEAP_TOP(c_p);
     FCALLS = c_p->fcalls;
     if (is_value(result)) {
	 r(0) = result;
	 CHECK_TERM(r(0));
	 NextPF(0, next);
     } else if (c_p->freason == TRAP) {
	 SET_CP(c_p, I+1);
	 SET_I(c_p->i);
	 SWAPIN;
	 r(0) = reg[0];
	 Dispatch();
     }
     goto find_func_info;
 }

    {
	Eterm element_index;
	Eterm element_tuple;

    OpCase(i_element_xjsd):
	element_tuple = xb(Arg(0));
	I++;
	goto do_element;

    OpCase(i_element_yjsd):
	element_tuple = yb(Arg(0));
	I++;
	goto do_element;

    OpCase(i_element_rjsd):
	element_tuple = r(0);
	/* Fall through */

    do_element:
	GetArg1(1, element_index);
	if (is_small(element_index) && is_tuple(element_tuple)) {
	    Eterm* tp = tuple_val(element_tuple);

	    if ((signed_val(element_index) >= 1) &&
		(signed_val(element_index) <= arityval(*tp))) {
		Eterm result = tp[signed_val(element_index)];
		StoreBifResult(2, result);
	    }
	}
    }
 /* Fall through */

 OpCase(badarg_j):
 badarg:
    c_p->freason = BADARG;
    goto lb_Cl_error;

    {
	Eterm fast_element_tuple;

    OpCase(i_fast_element_rjId):
	fast_element_tuple = r(0);

    do_fast_element:
	if (is_tuple(fast_element_tuple)) {
	    Eterm* tp = tuple_val(fast_element_tuple);
	    Eterm pos = Arg(1);	/* Untagged integer >= 1 */
	    if (pos <= arityval(*tp)) {
		Eterm result = tp[pos];
		StoreBifResult(2, result);
	    }
	}
     goto badarg;

    OpCase(i_fast_element_xjId):
     fast_element_tuple = xb(Arg(0));
     I++;
     goto do_fast_element;

    OpCase(i_fast_element_yjId):
     fast_element_tuple = yb(Arg(0));
     I++;
     goto do_fast_element;
 }

 OpCase(catch_yf):
     c_p->catches++;
     yb(Arg(0)) = Arg(1);
     Next(2);

 OpCase(catch_end_y): {
     c_p->catches--;
     make_blank(yb(Arg(0)));
     if (is_non_value(r(0))) {
	 if (x(1) == am_throw) {
	     r(0) = x(2);
	 } else {
	     if (x(1) == am_error) {
	         SWAPOUT;
		 x(2) = add_stacktrace(c_p, x(2), x(3));
		 SWAPIN;
	     }
	     /* only x(2) is included in the rootset here */
	     if (E - HTOP < 3 || c_p->mbuf) {	/* Force GC in case add_stacktrace()
						 * created heap fragments */
		 SWAPOUT;
		 PROCESS_MAIN_CHK_LOCKS(c_p);
		 FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
		 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
		 PROCESS_MAIN_CHK_LOCKS(c_p);
		 SWAPIN;
	     }
	     r(0) = TUPLE2(HTOP, am_EXIT, x(2));
	     HTOP += 3;
	 }
     }
     CHECK_TERM(r(0));
     Next(1);
 }

 OpCase(try_end_y): {
     c_p->catches--;
     make_blank(yb(Arg(0)));
     if (is_non_value(r(0))) {
	 r(0) = x(1);
	 x(1) = x(2);
	 x(2) = x(3);
     }
     Next(1);
 }

 /*
  * Skeleton for receive statement:
  *
  *             recv_mark L1                     Optional
  *             call make_ref/monitor            Optional
  *             ...
  *             recv_set L1                      Optional
  *      L1:          <-------------------+
  *                   <-----------+       |
  *     	     	       	  |   	  |
  *             loop_rec L2 ------+---+   |
  *             ...               |   |   |
  *             remove_message 	  |   |	  |
  *             jump L3           |   |   |
  *		...	          |   |   |
  *		loop_rec_end L1 --+   |   |
  *      L2:          <---------------+   |
  *	   	wait L1  -----------------+      or wait_timeout
  *		timeout
  *
  *	 L3:    Code after receive...
  *
  *
  */

 OpCase(recv_mark_f): {
     /*
      * Save the current position in message buffer and the
      * the label for the loop_rec/2 instruction for the
      * the receive statement.
      */
     c_p->msg.mark = (BeamInstr *) Arg(0);
     c_p->msg.saved_last = c_p->msg.last;
     Next(1);
 }

 OpCase(i_recv_set): {
     /*
      * If the mark is valid (points to the loop_rec/2
      * instruction that follows), we know that the saved
      * position points to the first message that could
      * possibly be matched out.
      *
      * If the mark is invalid, we do nothing, meaning that
      * we will look through all messages in the message queue.
      */
     if (c_p->msg.mark == (BeamInstr *) (I+1)) {
	 c_p->msg.save = c_p->msg.saved_last;
     }
     I++;
     /* Fall through to the loop_rec/2 instruction */
 }

    /*
     * Pick up the next message and place it in x(0).
     * If no message, jump to a wait or wait_timeout instruction.
     */
 OpCase(i_loop_rec_fr):
 {
     BeamInstr *next;
     ErlMessage* msgp;

 loop_rec__:

     PROCESS_MAIN_CHK_LOCKS(c_p);

     msgp = PEEK_MESSAGE(c_p);

     if (!msgp) {
#ifdef ERTS_SMP
	 erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
	 /* Make sure messages wont pass exit signals... */
	 if (ERTS_PROC_PENDING_EXIT(c_p)) {
	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
	     SWAPOUT;
	     goto do_schedule; /* Will be rescheduled for exit */
	 }
	 ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
	 msgp = PEEK_MESSAGE(c_p);
	 if (msgp)
	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
	 else
#endif
# 1946 "beam/beam_emu.c"
	 {
	     SET_I((BeamInstr *) Arg(0));
	     Goto(*I);		/* Jump to a wait or wait_timeout instruction */
	 }
     }
     ErtsMoveMsgAttachmentIntoProc(msgp, c_p, E, HTOP, FCALLS,
				   {
				       SWAPOUT;
				       reg[0] = r(0);
				       PROCESS_MAIN_CHK_LOCKS(c_p);
				   },
				   {
				       ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
				       PROCESS_MAIN_CHK_LOCKS(c_p);
				       r(0) = reg[0];
				       SWAPIN;
				   });
     if (is_non_value(ERL_MESSAGE_TERM(msgp))) {
	 /*
	  * A corrupt distribution message that we weren't able to decode;
	  * remove it...
	  */
	 ASSERT(!msgp->data.attached);
         /* TODO: Add DTrace probe for this bad message situation? */
	 UNLINK_MESSAGE(c_p, msgp);
	 free_message(msgp);
	 goto loop_rec__;
     }
     PreFetch(1, next);
     r(0) = ERL_MESSAGE_TERM(msgp);
     NextPF(1, next);
 }

 /*
  * Remove a (matched) message from the message queue.
  */
 OpCase(remove_message): {
     BeamInstr *next;
     ErlMessage* msgp;

     PROCESS_MAIN_CHK_LOCKS(c_p);

     PreFetch(0, next);
     msgp = PEEK_MESSAGE(c_p);

     if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
	 save_calls(c_p, &exp_receive);
     }
     if (ERL_MESSAGE_TOKEN(msgp) == NIL) {
#ifdef USE_VM_PROBES
	 if (DT_UTAG(c_p) != NIL) {
	     if (DT_UTAG_FLAGS(c_p) & DT_UTAG_PERMANENT) {
		 SEQ_TRACE_TOKEN(c_p) = am_have_dt_utag;
#ifdef DTRACE_TAG_HARDDEBUG
		 if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) 
		     erts_fprintf(stderr,
				  "Dtrace -> (%T) stop spreading "
				  "tag %T with message %T\r\n",
				  c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
#endif
# 2006 "beam/beam_emu.c"
	     } else {
#ifdef DTRACE_TAG_HARDDEBUG
		 erts_fprintf(stderr,
			      "Dtrace -> (%T) kill tag %T with "
			      "message %T\r\n",
			      c_p->common.id,DT_UTAG(c_p),ERL_MESSAGE_TERM(msgp));
#endif
# 2013 "beam/beam_emu.c"
		 DT_UTAG(c_p) = NIL;
		 SEQ_TRACE_TOKEN(c_p) = NIL;
	     }
	 } else {
#endif
# 2018 "beam/beam_emu.c"
	     SEQ_TRACE_TOKEN(c_p) = NIL;
#ifdef USE_VM_PROBES
	 }
	 DT_UTAG_FLAGS(c_p) &= ~DT_UTAG_SPREADING;
#endif
# 2023 "beam/beam_emu.c"
     } else if (ERL_MESSAGE_TOKEN(msgp) != am_undefined) {
	 Eterm msg;
	 SEQ_TRACE_TOKEN(c_p) = ERL_MESSAGE_TOKEN(msgp);
#ifdef USE_VM_PROBES
	 if (ERL_MESSAGE_TOKEN(msgp) == am_have_dt_utag) {
	     if (DT_UTAG(c_p) == NIL) {
		 DT_UTAG(c_p) = ERL_MESSAGE_DT_UTAG(msgp);
	     }
	     DT_UTAG_FLAGS(c_p) |= DT_UTAG_SPREADING;
#ifdef DTRACE_TAG_HARDDEBUG
	     erts_fprintf(stderr,
			  "Dtrace -> (%T) receive tag (%T) "
			  "with message %T\r\n",
			  c_p->common.id, DT_UTAG(c_p), ERL_MESSAGE_TERM(msgp));
#endif
# 2038 "beam/beam_emu.c"
	 } else {
#endif
# 2040 "beam/beam_emu.c"
	     ASSERT(is_tuple(SEQ_TRACE_TOKEN(c_p)));
	     ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5);
	     ASSERT(is_small(SEQ_TRACE_TOKEN_SERIAL(c_p)));
	     ASSERT(is_small(SEQ_TRACE_TOKEN_LASTCNT(c_p)));
	     ASSERT(is_small(SEQ_TRACE_TOKEN_FLAGS(c_p)));
	     ASSERT(is_pid(SEQ_TRACE_TOKEN_SENDER(c_p)));
	     c_p->seq_trace_lastcnt = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
	     if (c_p->seq_trace_clock < unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p))) {
		 c_p->seq_trace_clock = unsigned_val(SEQ_TRACE_TOKEN_SERIAL(c_p));
	     }
	     msg = ERL_MESSAGE_TERM(msgp);
	     seq_trace_output(SEQ_TRACE_TOKEN(c_p), msg, SEQ_TRACE_RECEIVE, 
			      c_p->common.id, c_p);
#ifdef USE_VM_PROBES
	 }
#endif
# 2056 "beam/beam_emu.c"
     }
#ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(message_receive)) {
         Eterm token2 = NIL;
         DTRACE_CHARBUF(receiver_name, DTRACE_TERM_BUF_SIZE);
         Sint tok_label = 0;
         Sint tok_lastcnt = 0;
         Sint tok_serial = 0;

         dtrace_proc_str(c_p, receiver_name);
         token2 = SEQ_TRACE_TOKEN(c_p);
         if (token2 != NIL && token2 != am_have_dt_utag) {
             tok_label = signed_val(SEQ_TRACE_T_LABEL(token2));
             tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(token2));
             tok_serial = signed_val(SEQ_TRACE_T_SERIAL(token2));
         }
         DTRACE6(message_receive,
                 receiver_name, size_object(ERL_MESSAGE_TERM(msgp)),
                 c_p->msg.len - 1, tok_label, tok_lastcnt, tok_serial);
     }
#endif
# 2077 "beam/beam_emu.c"
     UNLINK_MESSAGE(c_p, msgp);
     JOIN_MESSAGE(c_p);
     CANCEL_TIMER(c_p);
     free_message(msgp);

     ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
     PROCESS_MAIN_CHK_LOCKS(c_p);

     NextPF(0, next);
 }

    /*
     * Advance the save pointer to the next message (the current
     * message didn't match), then jump to the loop_rec instruction.
     */
 OpCase(loop_rec_end_f): {
     SET_I((BeamInstr *) Arg(0));
     SAVE_MESSAGE(c_p);
     goto loop_rec__;
 }
    /*
     * Prepare to wait for a message or a timeout, whichever occurs first.
     *
     * Note: In order to keep the compatibility between 32 and 64 bits
     * emulators, only timeout values that can be represented in 32 bits
     * (unsigned) or less are allowed.
     */


 OpCase(i_wait_timeout_fs): {
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);

     /* Fall through */
 }
 OpCase(i_wait_timeout_locked_fs): {
     Eterm timeout_value;

     /*
      * If we have already set the timer, we must NOT set it again.  Therefore,
      * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
      */
     if (c_p->flags & (F_INSLPQUEUE | F_TIMO)) {
	 goto wait2;
     }
     GetArg1(1, timeout_value);
     if (timeout_value != make_small(0)) {

	 if (timeout_value == am_infinity)
	     c_p->flags |= F_TIMO;
	 else {
	     int tres = erts_set_proc_timer_term(c_p, timeout_value);
	     if (tres == 0) {
		 /*
		  * The timer routiner will set c_p->i to the value in
		  * c_p->def_arg_reg[0].  Note that it is safe to use this
		  * location because there are no living x registers in
		  * a receive statement.
		  * Note that for the halfword emulator, the two first elements
		  * of the array are used.
		  */
		 BeamInstr** pi = (BeamInstr**) c_p->def_arg_reg;
		 *pi = I+3;
	     }
	     else { /* Wrong time */
	     OpCase(i_wait_error_locked): {
		     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
		     /* Fall through */
		 }
	     OpCase(i_wait_error): {
		     c_p->freason = EXC_TIMEOUT_VALUE;
		     goto find_func_info;
		 }
	     }
	 }

	 /*
	  * Prepare to wait indefinitely for a new message to arrive
	  * (or the time set above if falling through from above).
	  *
	  * When a new message arrives, control will be transferred
	  * the loop_rec instruction (at label L1).  In case of
	  * of timeout, control will be transferred to the timeout
	  * instruction following the wait_timeout instruction.
	  */

	 OpCase(wait_locked_f):
	 OpCase(wait_f):

	 wait2: {
#ifndef ERTS_SMP
	     if (ERTS_PROC_IS_EXITING(c_p)) {
		 /*
		  * I non smp case:
		  *
		  * Currently executing process might be sent an exit
		  * signal if it is traced by a port that it also is
		  * linked to, and the port terminates during the
		  * trace. In this case we do *not* want to clear
		  * the active flag, which will make the process hang
		  * in limbo forever.
		  */
		 SWAPOUT;
		 goto do_schedule;
	     }
#endif
# 2182 "beam/beam_emu.c"
	     c_p->i = (BeamInstr *) Arg(0); /* L1 */
	     SWAPOUT;
	     c_p->arity = 0;
	     erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
	     ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
	     c_p->current = NULL;
	     goto do_schedule;
	 }
	 OpCase(wait_unlocked_f): {
	     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
	     goto wait2;
	 }
     }
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
     Next(2);
 }

 OpCase(i_wait_timeout_fI): {
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 }

 OpCase(i_wait_timeout_locked_fI):
 {
     /*
      * If we have already set the timer, we must NOT set it again.  Therefore,
      * we must test the F_INSLPQUEUE flag as well as the F_TIMO flag.
      */
     if ((c_p->flags & (F_INSLPQUEUE | F_TIMO)) == 0) {
	 BeamInstr** p = (BeamInstr **) c_p->def_arg_reg;
	 *p = I+3;
	 erts_set_proc_timer_uword(c_p, Arg(1));
     }
     goto wait2;
 }

    /*
     * A timeout has occurred.  Reset the save pointer so that the next
     * receive statement will examine the first message first.
     */
 OpCase(timeout_locked): {
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCKS_MSG_RECEIVE);
 }

 OpCase(timeout): {
     BeamInstr *next;

     PreFetch(0, next);
     if (IS_TRACED_FL(c_p, F_TRACE_RECEIVE)) {
	 trace_receive(c_p, am_timeout);
     }
     if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
	 save_calls(c_p, &exp_timeout);
     }
     c_p->flags &= ~F_TIMO;
     JOIN_MESSAGE(c_p);
     NextPF(0, next);
 }

 {
     Eterm select_val2;

 OpCase(i_select_tuple_arity2_yfAAff):
     select_val2 = yb(Arg(0));
     goto do_select_tuple_arity2;

 OpCase(i_select_tuple_arity2_xfAAff):
     select_val2 = xb(Arg(0));
     goto do_select_tuple_arity2;

 OpCase(i_select_tuple_arity2_rfAAff):
     select_val2 = r(0);
     I--;

 do_select_tuple_arity2:
     if (is_not_tuple(select_val2)) {
	 goto select_val2_fail;
     }
     select_val2 = *tuple_val(select_val2);
     goto do_select_val2;

 OpCase(i_select_val2_yfccff):
     select_val2 = yb(Arg(0));
     goto do_select_val2;

 OpCase(i_select_val2_xfccff):
     select_val2 = xb(Arg(0));
     goto do_select_val2;

 OpCase(i_select_val2_rfccff):
     select_val2 = r(0);
     I--;

 do_select_val2:
     if (select_val2 == Arg(2)) {
	 I += 3;
     } else if (select_val2 == Arg(3)) {
	 I += 4;
     }

 select_val2_fail:
     SET_I((BeamInstr *) Arg(1));
     Goto(*I);
 }

 {
     Eterm select_val;

 OpCase(i_select_tuple_arity_xfI):
     select_val = xb(Arg(0));
     goto do_select_tuple_arity;

 OpCase(i_select_tuple_arity_yfI):
     select_val = yb(Arg(0));
     goto do_select_tuple_arity;

 OpCase(i_select_tuple_arity_rfI):
     select_val = r(0);
     I--;

 do_select_tuple_arity:
     if (is_tuple(select_val)) {
	 select_val = *tuple_val(select_val);
	 goto do_linear_search;
     }
     SET_I((BeamInstr *) Arg(1));
     Goto(*I);

 OpCase(i_select_val_lins_xfI):
     select_val = xb(Arg(0));
     goto do_linear_search;

 OpCase(i_select_val_lins_yfI):
     select_val = yb(Arg(0));
     goto do_linear_search;

 OpCase(i_select_val_lins_rfI):
     select_val = r(0);
     I--;

 do_linear_search: {
     BeamInstr *vs = &Arg(3);
     int ix = 0;

     for(;;) {
	 if (vs[ix+0] >= select_val) { ix += 0; break; }
	 if (vs[ix+1] >= select_val) { ix += 1; break; }
	 ix += 2;
     }

     if (vs[ix] == select_val) {
	 I += ix + Arg(2) + 2;
     }

     SET_I((BeamInstr *) Arg(1));
     Goto(*I);
   }

 OpCase(i_select_val_bins_xfI):
     select_val = xb(Arg(0));
     goto do_binary_search;

 OpCase(i_select_val_bins_yfI):
     select_val = yb(Arg(0));
     goto do_binary_search;
     
 OpCase(i_select_val_bins_rfI):
     select_val = r(0);
     I--;

 do_binary_search:
 {
     struct Pairs {
	 BeamInstr val;
	 BeamInstr* addr;
     };
     struct Pairs* low;
     struct Pairs* high;
     struct Pairs* mid;
     int bdiff; /* int not long because the arrays aren't that large */

     low = (struct Pairs *) &Arg(3);
     high = low + Arg(2);

     /* The pointer subtraction (high-low) below must produce
      * a signed result, because high could be < low. That
      * requires the compiler to insert quite a bit of code.
      *
      * However, high will be > low so the result will be
      * positive. We can use that knowledge to optimise the
      * entire sequence, from the initial comparison to the
      * computation of mid.
      *
      * -- Mikael Pettersson, Acumem AB
      *
      * Original loop control code:
      *
      * while (low < high) {
      *    mid = low + (high-low) / 2;
      *
      */
     while ((bdiff = (int)((char*)high - (char*)low)) > 0) {
	 unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1);

	 mid = (struct Pairs*)((char*)low + boffset);
	 if (select_val < mid->val) {
	     high = mid;
	 } else if (select_val > mid->val) {
	     low = mid + 1;
	 } else {
	     SET_I(mid->addr);
	     Goto(*I);
	 }
     }
     SET_I((BeamInstr *) Arg(1));
     Goto(*I);
 }
 }

 {
     Eterm jump_on_val_zero_index;
     
 OpCase(i_jump_on_val_zero_yfI):
     jump_on_val_zero_index = yb(Arg(0));
     goto do_jump_on_val_zero_index;

 OpCase(i_jump_on_val_zero_xfI):
     jump_on_val_zero_index = xb(Arg(0));
     goto do_jump_on_val_zero_index;

 OpCase(i_jump_on_val_zero_rfI):
     jump_on_val_zero_index = r(0);
     I--;

 do_jump_on_val_zero_index:
     if (is_small(jump_on_val_zero_index)) {
	 jump_on_val_zero_index = signed_val(jump_on_val_zero_index);
	 if (jump_on_val_zero_index < Arg(2)) {
	     SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]);
	     Goto(*I);
	 }
     }
     SET_I((BeamInstr *) Arg(1));
     Goto(*I);
 }

 {
     Eterm jump_on_val_index;

 
 OpCase(i_jump_on_val_yfII):
     jump_on_val_index = yb(Arg(0));
     goto do_jump_on_val_index;

 OpCase(i_jump_on_val_xfII):
     jump_on_val_index = xb(Arg(0));
     goto do_jump_on_val_index;

 OpCase(i_jump_on_val_rfII):
     jump_on_val_index = r(0);
     I--;

 do_jump_on_val_index:
     if (is_small(jump_on_val_index)) {
	 jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3));
	 if (jump_on_val_index < Arg(2)) {
	     SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]);
	     Goto(*I);
	 }
     }
     SET_I((BeamInstr *) Arg(1));
     Goto(*I);
 }

 do_put_tuple: {
     Eterm* hp = HTOP;

     *hp++ = make_arityval(pt_arity);

     do {
	 Eterm term = *I++;
	 switch (term & _TAG_IMMED1_MASK) {
	 case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
	     *hp++ = r(0);
	     break;
	 case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
	     *hp++ = x(term >> _TAG_IMMED1_SIZE);
	     break;
	 case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:
	     *hp++ = y(term >> _TAG_IMMED1_SIZE);
	     break;
	 default:
	     *hp++ = term;
	     break;
	 }
     } while (--pt_arity != 0);
     HTOP = hp;
     Goto(*I);
 }

 OpCase(new_map_dII): {
     Eterm res;

     x(0) = r(0);
     SWAPOUT;
     res = new_map(c_p, reg, I-1);
     SWAPIN;
     r(0) = x(0);
     StoreResult(res, Arg(0));
     Next(3+Arg(2));
 }

#define PUT_TERM_REG(term, desc)				\
do {								\
    switch ((desc) & _TAG_IMMED1_MASK) {			\
    case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
	r(0) = (term);						\
	break;							\
    case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
	x((desc) >> _TAG_IMMED1_SIZE) = (term);			\
	break;							\
    case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
	y((desc) >> _TAG_IMMED1_SIZE) = (term);			\
	break;							\
    default:							\
	ASSERT(0);						\
	break;							\
    }								\
} while(0)

 OpCase(i_get_map_elements_fsI): {
    Eterm map;
    BeamInstr *fs;
    Uint sz, n;

    GetArg1(1, map);

    /* this instruction assumes Arg1 is a map,
     * i.e. that it follows a test is_map if needed.
     */

    n  = (Uint)Arg(2) / 3;
    fs = &Arg(3); /* pattern fields and target registers */

    if (is_flatmap(map)) {
	flatmap_t *mp;
	Eterm *ks;
	Eterm *vs;

	mp = (flatmap_t *)flatmap_val(map);
	sz = flatmap_get_size(mp);

	if (sz == 0) {
	    ClauseFail();
	}

	ks = flatmap_get_keys(mp);
	vs = flatmap_get_values(mp);

	while(sz) {
	    if (EQ((Eterm) fs[0], *ks)) {
		PUT_TERM_REG(*vs, fs[1]);
		n--;
		fs += 3;
		/* no more values to fetch, we are done */
		if (n == 0) {
		    I = fs;
		    Next(-1);
		}
	    }
	    ks++, sz--, vs++;
	}

	ClauseFail();
    } else {
	const Eterm *v;
	Uint32 hx;
	ASSERT(is_hashmap(map));
	while(n--) {
	    hx = fs[2];
	    ASSERT(hx == hashmap_make_hash((Eterm)fs[0]));
	    if ((v = erts_hashmap_get(hx, (Eterm)fs[0], map)) == NULL) {
		ClauseFail();
	    }
	    PUT_TERM_REG(*v, fs[1]);
	    fs += 3;
	}
	I = fs;
	Next(-1);
    }
 }
#undef PUT_TERM_REG

 OpCase(update_map_assoc_jsdII): {
     Eterm res;
     Eterm map;

     GetArg1(1, map);
     x(0) = r(0);
     SWAPOUT;
     res = update_map_assoc(c_p, reg, map, I);
     SWAPIN;
     if (is_value(res)) {
	 r(0) = x(0);
	 StoreResult(res, Arg(2));
	 Next(5+Arg(4));
     } else {
	 /*
	  * This can only happen if the code was compiled
	  * with the compiler in OTP 17.
	  */
	 c_p->freason = BADMAP;
	 c_p->fvalue = map;
	 goto lb_Cl_error;
     }
 }

 OpCase(update_map_exact_jsdII): {
     Eterm res;
     Eterm map;

     GetArg1(1, map);
     x(0) = r(0);
     SWAPOUT;
     res = update_map_exact(c_p, reg, map, I);
     SWAPIN;
     if (is_value(res)) {
	 r(0) = x(0);
	 StoreResult(res, Arg(2));
	 Next(5+Arg(4));
     } else {
	 goto lb_Cl_error;
     }
 }


    /*
     * All guards with zero arguments have special instructions:
     * 	self/0
     * 	node/0
     *
     * All other guard BIFs take one or two arguments.
     */

    /*
     * Guard BIF in head.  On failure, ignore the error and jump
     * to the code for the next clause.  We don't support tracing
     * of guard BIFs.
     */

 OpCase(bif1_fbsd):
    {
	Eterm (*bf)(Process*, Eterm*);
	Eterm tmp_reg[1];
	Eterm result;

	GetArg1(2, tmp_reg[0]);
	bf = (BifFunction) Arg(1);
	c_p->fcalls = FCALLS;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	result = (*bf)(c_p, tmp_reg);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_HOLE_CHECK(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    StoreBifResult(3, result);
	}
	SET_I((BeamInstr *) Arg(0));
	Goto(*I);
    }

    /*
     * Guard BIF in body.  It can fail like any BIF.  No trace support.
     */

 OpCase(bif1_body_bsd):
    {
	Eterm (*bf)(Process*, Eterm*);

	Eterm tmp_reg[1];
	Eterm result;

	GetArg1(1, tmp_reg[0]);
	bf = (BifFunction) Arg(0);
	c_p->fcalls = FCALLS;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	result = (*bf)(c_p, tmp_reg);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_HOLE_CHECK(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    StoreBifResult(2, result);
	}
	reg[0] = tmp_reg[0];
	SWAPOUT;
	I = handle_error(c_p, I, reg, bf);
	goto post_error_handling;
    }

 OpCase(i_gc_bif1_jIsId):
    {
	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
	GcBifFunction bf;
	Eterm arg;
	Eterm result;
	Uint live = (Uint) Arg(3);

	GetArg1(2, arg);
	reg[0] = r(0);
	reg[live] = arg;
	bf = (GcBifFunction) Arg(1);
	c_p->fcalls = FCALLS;
	SWAPOUT;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
	result = (*bf)(c_p, reg, live);
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	SWAPIN;
	r(0) = reg[0];
	ERTS_HOLE_CHECK(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    StoreBifResult(4, result);
	}
	if (Arg(0) != 0) {
	    SET_I((BeamInstr *) Arg(0));
	    Goto(*I);
	}
	reg[0] = arg;
	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
	goto post_error_handling;
    }

 OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1
			    and i_gc_bif3 */
    {
	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
	GcBifFunction bf;
	Eterm result;
	Uint live = (Uint) Arg(2);

	reg[0] = r(0);
	reg[live++] = tmp_arg1;
	reg[live] = tmp_arg2;
	bf = (GcBifFunction) Arg(1);
	c_p->fcalls = FCALLS;
	SWAPOUT;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
	result = (*bf)(c_p, reg, live);
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	SWAPIN;
	r(0) = reg[0];
	ERTS_HOLE_CHECK(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    StoreBifResult(3, result);
	}
	if (Arg(0) != 0) {
	    SET_I((BeamInstr *) Arg(0));
	    Goto(*I);
	}
	reg[0] = tmp_arg1;
	reg[1] = tmp_arg2;
	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
	goto post_error_handling;
    }

 OpCase(i_gc_bif3_jIsId):
    {
	typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint);
	GcBifFunction bf;
	Eterm arg;
	Eterm result;
	Uint live = (Uint) Arg(3);

	GetArg1(2, arg);
	reg[0] = r(0);
	reg[live++] = arg;
	reg[live++] = tmp_arg1;
	reg[live] = tmp_arg2;
	bf = (GcBifFunction) Arg(1);
	c_p->fcalls = FCALLS;
	SWAPOUT;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
	result = (*bf)(c_p, reg, live);
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	SWAPIN;
	r(0) = reg[0];
	ERTS_HOLE_CHECK(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    StoreBifResult(4, result);
	}
	if (Arg(0) != 0) {
	    SET_I((BeamInstr *) Arg(0));
	    Goto(*I);
	}
	reg[0] = arg;
	reg[1] = tmp_arg1;
	reg[2] = tmp_arg2;
	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
	goto post_error_handling;
    }

 /*
  * Guards bifs and, or, xor in guards.
  */
 OpCase(i_bif2_fbd):
    {
	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
	Eterm (*bf)(Process*, Eterm*);
	Eterm result;

	bf = (BifFunction) Arg(1);
	c_p->fcalls = FCALLS;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	result = (*bf)(c_p, tmp_reg);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_HOLE_CHECK(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    StoreBifResult(2, result);
	}
	SET_I((BeamInstr *) Arg(0));
	Goto(*I);
    }

 /*
  * Guards bifs and, or, xor, relational operators in body.
  */
 OpCase(i_bif2_body_bd):
    {
	Eterm tmp_reg[2] = {tmp_arg1, tmp_arg2};
	Eterm (*bf)(Process*, Eterm*);
	Eterm result;

	bf = (BifFunction) Arg(0);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	result = (*bf)(c_p, tmp_reg);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	ERTS_HOLE_CHECK(c_p);
	if (is_value(result)) {
	    ASSERT(!is_CP(result));
	    StoreBifResult(1, result);
	}
	reg[0] = tmp_arg1;
	reg[1] = tmp_arg2;
	SWAPOUT;
	I = handle_error(c_p, I, reg, bf);
	goto post_error_handling;
    }

    /*
     * The most general BIF call.  The BIF may build any amount of data
     * on the heap.  The result is always returned in r(0).
     */
 OpCase(call_bif_e):
    {
	Eterm (*bf)(Process*, Eterm*, BeamInstr*) = GET_BIF_ADDRESS(Arg(0));
	Eterm result;
	BeamInstr *next;

	PRE_BIF_SWAPOUT(c_p);
	c_p->fcalls = FCALLS - 1;
	if (FCALLS <= 0) {
	   save_calls(c_p, (Export *) Arg(0));
	}
	PreFetch(1, next);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	reg[0] = r(0);
	result = (*bf)(c_p, reg, I);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	ERTS_HOLE_CHECK(c_p);
	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	if (c_p->mbuf || MSO(c_p).overhead >= BIN_VHEAP_SZ(c_p)) {
	    Uint arity = ((Export *)Arg(0))->code[2];
	    result = erts_gc_after_bif_call(c_p, result, reg, arity);
	    E = c_p->stop;
	}
	HTOP = HEAP_TOP(c_p);
	FCALLS = c_p->fcalls;
	if (is_value(result)) {
	    r(0) = result;
	    CHECK_TERM(r(0));
	    NextPF(1, next);
	} else if (c_p->freason == TRAP) {
	    SET_CP(c_p, I+2);
	    SET_I(c_p->i);
	    SWAPIN;
	    r(0) = reg[0];
	    Dispatch();
	}

	/*
	 * Error handling.  SWAPOUT is not needed because it was done above.
	 */
	ASSERT(c_p->stop == E);
	I = handle_error(c_p, I, reg, bf);
	goto post_error_handling;
    }

 /*
  * Arithmetic operations.
  */

 OpCase(i_times_jId):
 {
     arith_func = ARITH_FUNC(mixed_times);
     goto do_big_arith2;
 }

 OpCase(i_m_div_jId):
 {
     arith_func = ARITH_FUNC(mixed_div);
     goto do_big_arith2;
 }

 OpCase(i_int_div_jId):
 {
     Eterm result;

     if (tmp_arg2 == SMALL_ZERO) {
	 goto badarith;
     } else if (is_both_small(tmp_arg1, tmp_arg2)) {
	 Sint ires = signed_val(tmp_arg1) / signed_val(tmp_arg2);
	 if (MY_IS_SSMALL(ires)) {
	     result = make_small(ires);
	     STORE_ARITH_RESULT(result);
	 }
     }
     arith_func = ARITH_FUNC(int_div);
     goto do_big_arith2;
 }

 OpCase(i_rem_jIxxd):
 {
     Eterm result;

     if (xb(Arg(3)) == SMALL_ZERO) {
	 goto badarith;
     } else if (is_both_small(xb(Arg(2)), xb(Arg(3)))) {
	 result = make_small(signed_val(xb(Arg(2))) % signed_val(xb(Arg(3))));
         StoreBifResult(4, result);
     }
     DO_BIG_ARITH(ARITH_FUNC(int_rem),xb(Arg(2)),xb(Arg(3)));
 }

 OpCase(i_rem_jId):
 {
     Eterm result;

     if (tmp_arg2 == SMALL_ZERO) {
	 goto badarith;
     } else if (is_both_small(tmp_arg1, tmp_arg2)) {
	 result = make_small(signed_val(tmp_arg1) % signed_val(tmp_arg2));
	 STORE_ARITH_RESULT(result);
     } else {
	 arith_func = ARITH_FUNC(int_rem);
	 goto do_big_arith2;
     }
 }

 OpCase(i_band_jIxcd):
 {
     Eterm result;

     if (is_both_small(xb(Arg(2)), Arg(3))) {
         /*
          * No need to untag -- TAG & TAG == TAG.
          */
         result = xb(Arg(2)) & Arg(3);
         StoreBifResult(4, result);
     }
     DO_BIG_ARITH(ARITH_FUNC(band),xb(Arg(2)),Arg(3));
 }

 OpCase(i_band_jId):
 {
     Eterm result;

     if (is_both_small(tmp_arg1, tmp_arg2)) {
	 /*
	  * No need to untag -- TAG & TAG == TAG.
	  */
	 result = tmp_arg1 & tmp_arg2;
	 STORE_ARITH_RESULT(result);
     }
     arith_func = ARITH_FUNC(band);
     goto do_big_arith2;
 }

#undef DO_BIG_ARITH

 do_big_arith2:
 {
     Eterm result;
     Uint live = Arg(1);

     SWAPOUT;
     reg[0] = r(0);
     reg[live] = tmp_arg1;
     reg[live+1] = tmp_arg2;
     result = arith_func(c_p, reg, live);
     r(0) = reg[0];
     SWAPIN;
     ERTS_HOLE_CHECK(c_p);
     if (is_value(result)) {
	 STORE_ARITH_RESULT(result);
     }
     goto lb_Cl_error;
 }

 /*
  * An error occured in an arithmetic operation or test that could
  * appear either in a head or in a body.
  * In a head, execution should continue at failure address in Arg(0).
  * In a body, Arg(0) == 0 and an exception should be raised.
  */
 lb_Cl_error: {
     if (Arg(0) != 0) {
	 OpCase(jump_f): {
	 jump_f:
	     SET_I((BeamInstr *) Arg(0));
	     Goto(*I);
	 }
     }
     ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue));
     goto find_func_info;
 }

 OpCase(i_bor_jId):
 {
     Eterm result;

     if (is_both_small(tmp_arg1, tmp_arg2)) {
	 /*
	  * No need to untag -- TAG | TAG == TAG.
	  */
	 result = tmp_arg1 | tmp_arg2;
	 STORE_ARITH_RESULT(result);
     }
     arith_func = ARITH_FUNC(bor);
     goto do_big_arith2;
 }

 OpCase(i_bxor_jId):
 {
     Eterm result;

     if (is_both_small(tmp_arg1, tmp_arg2)) {
	 /*
	  * We could extract the tag from one argument, but a tag extraction
	  * could mean a shift.  Therefore, play it safe here.
	  */
	 result = make_small(signed_val(tmp_arg1) ^ signed_val(tmp_arg2));
	 STORE_ARITH_RESULT(result);
     }
     arith_func = ARITH_FUNC(bxor);
     goto do_big_arith2;
 }

 {
     Sint i;
     Sint ires;
     Eterm* bigp;

     OpCase(i_bsr_jId):
	 if (is_small(tmp_arg2)) {
	     i = -signed_val(tmp_arg2);
	     if (is_small(tmp_arg1)) {
		 goto small_shift;
	     } else if (is_big(tmp_arg1)) {
		 if (i == 0) {
		     StoreBifResult(2, tmp_arg1);
		 }
		 goto big_shift;
	     }
	 } else if (is_big(tmp_arg2)) {
	     /*
	      * N bsr NegativeBigNum == N bsl MAX_SMALL
	      * N bsr PositiveBigNum == N bsl MIN_SMALL
	      */
	     tmp_arg2 = make_small(bignum_header_is_neg(*big_val(tmp_arg2)) ?
				   MAX_SMALL : MIN_SMALL);
	     goto do_bsl;
	}
     goto badarith;
     
     OpCase(i_bsl_jId):
 do_bsl:
	 if (is_small(tmp_arg2)) {
	     i = signed_val(tmp_arg2);

	     if (is_small(tmp_arg1)) {
	     small_shift:
		 ires = signed_val(tmp_arg1);
	     
		 if (i == 0 || ires == 0) {
		     StoreBifResult(2, tmp_arg1);
		 } else if (i < 0)  { /* Right shift */
		     i = -i;
		     if (i >= SMALL_BITS-1) {
			 tmp_arg1 = (ires < 0) ? SMALL_MINUS_ONE : SMALL_ZERO;
		     } else {
			 tmp_arg1 = make_small(ires >> i);
		     }
		     StoreBifResult(2, tmp_arg1);
		 } else if (i < SMALL_BITS-1) { /* Left shift */
		     if ((ires > 0 && ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ires) == 0) ||
			 ((~(Uint)0 << ((SMALL_BITS-1)-i)) & ~ires) == 0) {
			 tmp_arg1 = make_small(ires << i);
			 StoreBifResult(2, tmp_arg1);
		     }
		 }
		 tmp_arg1 = small_to_big(ires, tmp_big);

	     big_shift:
		 if (i > 0) {	/* Left shift. */
		     ires = big_size(tmp_arg1) + (i / D_EXP);
		 } else {	/* Right shift. */
		     ires = big_size(tmp_arg1);
		     if (ires <= (-i / D_EXP))
			 ires = 3; /* ??? */
		     else
			 ires -= (-i / D_EXP);
		 }
		 {
		     ires = BIG_NEED_SIZE(ires+1);
		     /*
		      * Slightly conservative check the size to avoid
		      * allocating huge amounts of memory for bignums that 
		      * clearly would overflow the arity in the header
		      * word.
		      */
		     if (ires-8 > BIG_ARITY_MAX) {
			 c_p->freason = SYSTEM_LIMIT;
			 goto lb_Cl_error;
		     }
		     TestHeapPreserve(ires+1, Arg(1), tmp_arg1);
		     bigp = HTOP;
		     tmp_arg1 = big_lshift(tmp_arg1, i, bigp);
		     if (is_big(tmp_arg1)) {
			 HTOP += bignum_header_arity(*HTOP) + 1;
		     }
		     HEAP_SPACE_VERIFIED(0);
		     if (is_nil(tmp_arg1)) {
			 /*
			  * This result must have been only slight larger
			  * than allowed since it wasn't caught by the
			  * previous test.
			  */
			 c_p->freason = SYSTEM_LIMIT;
			 goto lb_Cl_error;
		     }
		     ERTS_HOLE_CHECK(c_p);
		     StoreBifResult(2, tmp_arg1);
		 }
	     } else if (is_big(tmp_arg1)) {
		 if (i == 0) {
		     StoreBifResult(2, tmp_arg1);
		 }
		 goto big_shift;
	     }
	 } else if (is_big(tmp_arg2)) {
	     if (bignum_header_is_neg(*big_val(tmp_arg2))) {
		 /*
		  * N bsl NegativeBigNum is either 0 or -1, depending on
		  * the sign of N. Since we don't believe this case
		  * is common, do the calculation with the minimum
		  * amount of code.
		  */
		 tmp_arg2 = make_small(MIN_SMALL);
		 goto do_bsl;
	     } else if (is_small(tmp_arg1) || is_big(tmp_arg1)) {
		 /*
		  * N bsl PositiveBigNum is too large to represent.
		  */
		 c_p->freason = SYSTEM_LIMIT;
		 goto lb_Cl_error;
	     }
	     /* Fall through if the left argument is not an integer. */
	 }
     /*
      * One or more non-integer arguments.
      */
     goto badarith;
 }

 OpCase(i_int_bnot_jsId):
 {
     Eterm bnot_val;

     GetArg1(1, bnot_val);
     if (is_small(bnot_val)) {
	 bnot_val = make_small(~signed_val(bnot_val));
     } else {
	 Uint live = Arg(2);
	 SWAPOUT;
	 reg[0] = r(0);
	 reg[live] = bnot_val;
	 bnot_val = erts_gc_bnot(c_p, reg, live);
	 r(0) = reg[0];
	 SWAPIN;
	 ERTS_HOLE_CHECK(c_p);
	 if (is_nil(bnot_val)) {
	     goto lb_Cl_error;
	 }
     }
     StoreBifResult(3, bnot_val);
 }

 badarith:
    c_p->freason = BADARITH;
    goto lb_Cl_error;

 OpCase(i_apply): {
     BeamInstr *next;
     SWAPOUT;
     next = apply(c_p, r(0), x(1), x(2), reg);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, I+1);
	 SET_I(next);
	 Dispatch();
     }
     I = handle_error(c_p, I, reg, apply_3);
     goto post_error_handling;
 }

 OpCase(i_apply_last_P): {
     BeamInstr *next;
     SWAPOUT;
     next = apply(c_p, r(0), x(1), x(2), reg);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
	 E = ADD_BYTE_OFFSET(E, Arg(0));
	 SET_I(next);
	 Dispatch();
     }
     I = handle_error(c_p, I, reg, apply_3);
     goto post_error_handling;
 }

 OpCase(i_apply_only): {
     BeamInstr *next;
     SWAPOUT;
     next = apply(c_p, r(0), x(1), x(2), reg);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_I(next);
	 Dispatch();
     }
     I = handle_error(c_p, I, reg, apply_3);
     goto post_error_handling;
 }

 OpCase(apply_I): {
     BeamInstr *next;

     reg[0] = r(0);
     SWAPOUT;
     next = fixed_apply(c_p, reg, Arg(0));
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, I+2);
	 SET_I(next);
	 Dispatch();
     }
     I = handle_error(c_p, I, reg, apply_3);
     goto post_error_handling;
 }

 OpCase(apply_last_IP): {
     BeamInstr *next;

     reg[0] = r(0);
     SWAPOUT;
     next = fixed_apply(c_p, reg, Arg(0));
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
	 E = ADD_BYTE_OFFSET(E, Arg(1));
	 SET_I(next);
	 Dispatch();
     }
     I = handle_error(c_p, I, reg, apply_3);
     goto post_error_handling;
 }

 OpCase(i_apply_fun): {
     BeamInstr *next;

     SWAPOUT;
     next = apply_fun(c_p, r(0), x(1), reg);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, I+1);
	 SET_I(next);
	 Dispatchfun();
     }
     goto find_func_info;
 }

 OpCase(i_apply_fun_last_P): {
     BeamInstr *next;

     SWAPOUT;
     next = apply_fun(c_p, r(0), x(1), reg);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
	 E = ADD_BYTE_OFFSET(E, Arg(0));
	 SET_I(next);
	 Dispatchfun();
     }
     goto find_func_info;
 }

 OpCase(i_apply_fun_only): {
     BeamInstr *next;

     SWAPOUT;
     next = apply_fun(c_p, r(0), x(1), reg);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_I(next);
	 Dispatchfun();
     }
     goto find_func_info;
 }

 OpCase(i_call_fun_I): {
     BeamInstr *next;

     SWAPOUT;
     reg[0] = r(0);

     next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
     SWAPIN;
     if (next != NULL) {
	 r(0) = reg[0];
	 SET_CP(c_p, I+2);
	 SET_I(next);
	 Dispatchfun();
     }
     goto find_func_info;
 }

 OpCase(i_call_fun_last_IP): {
     BeamInstr *next;

     SWAPOUT;
     reg[0] = r(0);
     next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
     SWAPIN;
     if (next != NULL) {
	r(0) = reg[0];
	SET_CP(c_p, (BeamInstr *) EXPAND_POINTER(E[0]));
	E = ADD_BYTE_OFFSET(E, Arg(1));
	SET_I(next);
	Dispatchfun();
     }
     goto find_func_info;
 }

#ifdef DEBUG
    /*
     * Set a breakpoint here to get control just after a call instruction.
     * I points to the first instruction in the called function.
     *
     * In gdb, use 'call dis(I-5, 1)' to show the name of the function.
     */
 do_dispatch:
     DispatchMacro();

 do_dispatchx:
     DispatchMacrox();

 do_dispatchfun:
     DispatchMacroFun();

#endif
# 3396 "beam/beam_emu.c"

    /*
     * Jumped to from the Dispatch() macro when the reductions are used up.
     *
     * Since the I register points just beyond the FuncBegin instruction, we
     * can get the module, function, and arity for the function being
     * called from I[-3], I[-2], and I[-1] respectively.
     */
 context_switch_fun:
    c_p->arity = I[-1] + 1;
    goto context_switch2;

 context_switch:
    c_p->arity = I[-1];

 context_switch2:		/* Entry for fun calls. */
    c_p->current = I-3;		/* Pointer to Mod, Func, Arity */

 {
     Eterm* argp;
     int i;

     /*
      * Make sure that there is enough room for the argument registers to be saved.
      */
     if (c_p->arity > c_p->max_arg_reg) {
	 /*
	  * Yes, this is an expensive operation, but you only pay it the first
	  * time you call a function with more than 6 arguments which is
	  * scheduled out.  This is better than paying for 26 words of wasted
	  * space for most processes which never call functions with more than
	  * 6 arguments.
	  */
	 Uint size = c_p->arity * sizeof(c_p->arg_reg[0]);
	 if (c_p->arg_reg != c_p->def_arg_reg) {
	     c_p->arg_reg = (Eterm *) erts_realloc(ERTS_ALC_T_ARG_REG,
						   (void *) c_p->arg_reg,
						   size);
	 } else {
	     c_p->arg_reg = (Eterm *) erts_alloc(ERTS_ALC_T_ARG_REG, size);
	 }
	 c_p->max_arg_reg = c_p->arity;
     }

     /*
      * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
      * now before saving registers.
      *
      * The '+ 1' compensates for the last increment which was not done
      * (beacuse the code for the Dispatch() macro becomes shorter that way).
      */

     reds_used = REDS_IN(c_p) - FCALLS + 1;
     
     /*
      * Save the argument registers and everything else.
      */

     argp = c_p->arg_reg;
     for (i = c_p->arity - 1; i > 0; i--) {
	 argp[i] = reg[i];
     }
     c_p->arg_reg[0] = r(0);
     SWAPOUT;
     c_p->i = I;
     goto do_schedule1;
 }

 OpCase(set_tuple_element_sdP): {
     Eterm element;
     Eterm tuple;
     BeamInstr *next;
     Eterm* p;
     
     PreFetch(3, next);
     GetArg2(0, element, tuple);
     ASSERT(is_tuple(tuple));
     p = (Eterm *) ((unsigned char *) tuple_val(tuple) + Arg(2));
     *p = element;
     NextPF(3, next);
 }

 OpCase(i_is_ne_exact_f):
    if (EQ(tmp_arg1, tmp_arg2)) {
	ClauseFail();
    }
    Next(1);

 OpCase(normal_exit): {
     SWAPOUT;
     c_p->freason = EXC_NORMAL;
     c_p->arity = 0;		/* In case this process will ever be garbed again. */
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     erts_do_exit_process(c_p, am_normal);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     goto do_schedule;
 }

 OpCase(continue_exit): {
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     erts_continue_exit_process(c_p);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     goto do_schedule;
 }

 OpCase(raise_ss): {
     /* This was not done very well in R10-0; then, we passed the tag in
	the first argument and hoped that the existing c_p->ftrace was
	still correct. But the ftrace-object already includes the tag
	(or rather, the freason). Now, we pass the original ftrace in
	the first argument. We also handle atom tags in the first
	argument for backwards compatibility.
     */
     Eterm raise_val1;
     Eterm raise_val2;
     GetArg2(0, raise_val1, raise_val2);
     c_p->fvalue = raise_val2;
     if (c_p->freason == EXC_NULL) {
       /* a safety check for the R10-0 case; should not happen */
       c_p->ftrace = NIL;
       c_p->freason = EXC_ERROR;
     }
     /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */
     switch (raise_val1) {
     case am_throw:
       c_p->freason = EXC_THROWN & ~EXF_SAVETRACE;
       break;
     case am_error:
       c_p->freason = EXC_ERROR & ~EXF_SAVETRACE;
       break;
     case am_exit:
       c_p->freason = EXC_EXIT & ~EXF_SAVETRACE;
       break;
     default:
       {/* R10-1 and later
	   XXX note: should do sanity check on given trace if it can be
	   passed from a user! Currently only expecting generated calls.
	*/
	 struct StackTrace *s;
	 c_p->ftrace = raise_val1;
	 s = get_trace_from_exc(raise_val1);
	 if (s == NULL) {
	   c_p->freason = EXC_ERROR;
	 } else {
	   c_p->freason = PRIMARY_EXCEPTION(s->freason);
	 }
       }
     }
     goto find_func_info;
 }

    {
	Eterm badmatch_val;

    OpCase(badmatch_y):
	badmatch_val = yb(Arg(0));
	goto do_badmatch;

    OpCase(badmatch_x):
	badmatch_val = xb(Arg(0));
	goto do_badmatch;

    OpCase(badmatch_r):
	badmatch_val = r(0);

    do_badmatch:
	c_p->fvalue = badmatch_val;
	c_p->freason = BADMATCH;
    }
 /* Fall through here */

 find_func_info: {
     reg[0] = r(0);
     SWAPOUT;
     I = handle_error(c_p, I, reg, NULL);
     goto post_error_handling;
 }

 OpCase(call_error_handler):
    /*
     * At this point, I points to the code[3] in the export entry for
     * a function which is not loaded.
     *
     * code[0]: Module
     * code[1]: Function
     * code[2]: Arity
     * code[3]: &&call_error_handler
     * code[4]: Not used
     */
    SWAPOUT;
    reg[0] = r(0);
    I = call_error_handler(c_p, I-3, reg, am_undefined_function);
    r(0) = reg[0];
    SWAPIN;
    if (I) {
	Goto(*I);
    }

 /* Fall through */
 OpCase(error_action_code): {
    handle_error:
     reg[0] = r(0);
     SWAPOUT;
     I = handle_error(c_p, NULL, reg, NULL);
 post_error_handling:
     if (I == 0) {
	 goto do_schedule;
     } else {
	 r(0) = reg[0];
	 ASSERT(!is_value(r(0)));
	 if (c_p->mbuf) {
	     erts_garbage_collect(c_p, 0, reg+1, 3);
	 }
	 SWAPIN;
	 Goto(*I);
     }
 }

    {
	Eterm nif_bif_result;
	Eterm bif_nif_arity;

    OpCase(call_nif):
	{
	    /*
	     * call_nif is always first instruction in function:
	     *
	     * I[-3]: Module
	     * I[-2]: Function
	     * I[-1]: Arity
	     * I[0]: &&call_nif
	     * I[1]: Function pointer to NIF function
	     * I[2]: Pointer to erl_module_nif
	     * I[3]: Function pointer to dirty NIF
	     */
	    BifFunction vbf;

	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
	    SWAPOUT;
	    c_p->fcalls = FCALLS - 1;
	    PROCESS_MAIN_CHK_LOCKS(c_p);
	    bif_nif_arity = I[-1];
	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);

	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
	    {
		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
		NifF* fp = vbf = (NifF*) I[1];
		struct enif_environment_t env;
		erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]);
		reg[0] = r(0);
		nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
		if (env.exception_thrown)
		    nif_bif_result = THE_NON_VALUE;
		erts_post_nif(&env);
	    }
	    ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result));
	    PROCESS_MAIN_CHK_LOCKS(c_p);
	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);

	    DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
	    goto apply_bif_or_nif_epilogue;
	 
	OpCase(apply_bif):
	    /*
	     * At this point, I points to the code[3] in the export entry for
	     * the BIF:
	     *
	     * code[0]: Module
	     * code[1]: Function
	     * code[2]: Arity
	     * code[3]: &&apply_bif
	     * code[4]: Function pointer to BIF function
	     */

	    c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
	    c_p->i = I;		/* In case we apply check_process_code/2. */
	    c_p->arity = 0;		/* To allow garbage collection on ourselves
					 * (check_process_code/2).
					 */
	    DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);

	    SWAPOUT;
	    c_p->fcalls = FCALLS - 1;
	    vbf = (BifFunction) Arg(0);
	    PROCESS_MAIN_CHK_LOCKS(c_p);
	    bif_nif_arity = I[-1];
            ASSERT(bif_nif_arity <= 4);
	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
	    reg[0] = r(0);
	    {
		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
		nif_bif_result = (*bf)(c_p, reg, I);
		ASSERT(!ERTS_PROC_IS_EXITING(c_p) ||
		       is_non_value(nif_bif_result));
		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
		PROCESS_MAIN_CHK_LOCKS(c_p);
	    }

	    DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);

	apply_bif_or_nif_epilogue:
	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
	    ERTS_HOLE_CHECK(c_p);
	    if (c_p->mbuf) {
		nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result,
						  reg, bif_nif_arity);
	    }
	    SWAPIN;  /* There might have been a garbage collection. */
	    FCALLS = c_p->fcalls;
	    if (is_value(nif_bif_result)) {
		r(0) = nif_bif_result;
		CHECK_TERM(r(0));
		SET_I(c_p->cp);
		c_p->cp = 0;
		Goto(*I);
	    } else if (c_p->freason == TRAP) {
		SET_I(c_p->i);
		r(0) = reg[0];
		if (c_p->flags & F_HIBERNATE_SCHED) {
		    c_p->flags &= ~F_HIBERNATE_SCHED;
		    goto do_schedule;
		}
		Dispatch();
	    }
	    I = handle_error(c_p, c_p->cp, reg, vbf);
	    goto post_error_handling;
	}
    }

 OpCase(i_get_sd):
    {
	Eterm arg;
	Eterm result;

	GetArg1(0, arg);
	result = erts_pd_hash_get(c_p, arg);
	StoreBifResult(1, result);
    }

    {
	Eterm case_end_val;

    OpCase(case_end_x):
	case_end_val = xb(Arg(0));
	goto do_case_end;

    OpCase(case_end_y):
	case_end_val = yb(Arg(0));
	goto do_case_end;

    OpCase(case_end_r):
	case_end_val = r(0);

    do_case_end:
	c_p->fvalue = case_end_val;
	c_p->freason = EXC_CASE_CLAUSE;
	goto find_func_info;
    }

 OpCase(if_end):
    c_p->freason = EXC_IF_CLAUSE;
    goto find_func_info;

 OpCase(i_func_info_IaaI): {
     c_p->freason = EXC_FUNCTION_CLAUSE;
     c_p->current = I + 2;
     goto handle_error;
 }

 OpCase(try_case_end_s):
    {
	Eterm try_case_end_val;
	GetArg1(0, try_case_end_val);
	c_p->fvalue = try_case_end_val;
	c_p->freason = EXC_TRY_CLAUSE;
	goto find_func_info;
    }

 /*
  * Construction of binaries using new instructions.
  */
 {
     Eterm new_binary;
     Eterm num_bits_term;
     Uint num_bits;
     Uint alloc;
     Uint num_bytes;

     OpCase(i_bs_init_bits_heap_IIId): {
	 num_bits = Arg(0);
	 alloc = Arg(1);
	 I++;
	 goto do_bs_init_bits_known;
     }
     
     OpCase(i_bs_init_bits_IId): {
	 num_bits = Arg(0);
	 alloc = 0;
	 goto do_bs_init_bits_known;
     }

     OpCase(i_bs_init_bits_fail_heap_IjId): {
	 /* tmp_arg1 was fetched by an i_fetch instruction */
	 num_bits_term = tmp_arg1;
	 alloc = Arg(0);
	 I++;
	 goto do_bs_init_bits;
     }

     OpCase(i_bs_init_bits_fail_rjId): {
	 num_bits_term = r(0);
	 alloc = 0;
	 goto do_bs_init_bits;
     }
     OpCase(i_bs_init_bits_fail_yjId): {
	 num_bits_term = yb(Arg(0));
	 I++;
	 alloc = 0;
	 goto do_bs_init_bits;
     }
     OpCase(i_bs_init_bits_fail_xjId): {
	 num_bits_term = xb(Arg(0));
	 I++;
	 alloc = 0;
     /* FALL THROUGH */
     }

     /* num_bits_term = Term for number of bits to build (small/big)
      * alloc = Number of words to allocate on heap
      * Operands: Fail Live Dst
      */

 do_bs_init_bits:
     if (is_small(num_bits_term)) {
	 Sint size = signed_val(num_bits_term);
	 if (size < 0) {
	     goto badarg;
	 }
	 num_bits = (Uint) size;
     } else {
	 Uint bits;

	 if (!term_to_Uint(num_bits_term, &bits)) {
	     c_p->freason = bits;
	     goto lb_Cl_error;

	 }
	 num_bits = (Eterm) bits;
     }

     /* num_bits = Number of bits to build
      * alloc = Number of extra words to allocate on heap
      * Operands: NotUsed Live Dst
      */
 do_bs_init_bits_known:
     num_bytes = ((Uint64)num_bits+(Uint64)7) >> 3;
     if (num_bits & 7) {
	 alloc += ERL_SUB_BIN_SIZE;
     }
     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
	 alloc += heap_bin_size(num_bytes);
     } else {
	 alloc += PROC_BIN_SIZE;
     }
     TestHeap(alloc, Arg(1));

     /* num_bits = Number of bits to build
      * num_bytes = Number of bytes to allocate in the binary
      * alloc = Total number of words to allocate on heap
      * Operands: NotUsed NotUsed Dst
      */
     if (num_bytes <= ERL_ONHEAP_BIN_LIMIT) {
	 ErlHeapBin* hb;

	 erts_bin_offset = 0;
	 erts_writable_bin = 0;
	 hb = (ErlHeapBin *) HTOP;
	 HTOP += heap_bin_size(num_bytes);
	 hb->thing_word = header_heap_bin(num_bytes);
	 hb->size = num_bytes;
	 erts_current_bin = (byte *) hb->data;
	 new_binary = make_binary(hb);

     do_bits_sub_bin:
	 if (num_bits & 7) {
	     ErlSubBin* sb;

	     sb = (ErlSubBin *) HTOP;
	     HTOP += ERL_SUB_BIN_SIZE;
	     sb->thing_word = HEADER_SUB_BIN;
	     sb->size = num_bytes - 1;
	     sb->bitsize = num_bits & 7;
	     sb->offs = 0;
	     sb->bitoffs = 0;
	     sb->is_writable = 0;
	     sb->orig = new_binary;
	     new_binary = make_binary(sb);
	 }
	 HEAP_SPACE_VERIFIED(0);
	 StoreBifResult(2, new_binary);
     } else {
	 Binary* bptr;
	 ProcBin* pb;

	 erts_bin_offset = 0;
	 erts_writable_bin = 0;

	 /*
	  * Allocate the binary struct itself.
	  */
	 bptr = erts_bin_nrml_alloc(num_bytes);
	 erts_refc_init(&bptr->refc, 1);
	 erts_current_bin = (byte *) bptr->orig_bytes;

	 /*
	  * Now allocate the ProcBin on the heap.
	  */
	 pb = (ProcBin *) HTOP;
	 HTOP += PROC_BIN_SIZE;
	 pb->thing_word = HEADER_PROC_BIN;
	 pb->size = num_bytes;
	 pb->next = MSO(c_p).first;
	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
	 pb->val = bptr;
	 pb->bytes = (byte*) bptr->orig_bytes;
	 pb->flags = 0;
	 OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm));
	 new_binary = make_binary(pb);
	 goto do_bits_sub_bin;
     }
 }

 {
     OpCase(i_bs_init_fail_heap_IjId): {
	 /* tmp_arg1 was fetched by an i_fetch instruction */
	 tmp_arg2 = Arg(0);
	 I++;
	 goto do_bs_init;
     }

     OpCase(i_bs_init_fail_rjId): {
	 tmp_arg1 = r(0);
	 tmp_arg2 = 0;
	 goto do_bs_init;
     }

     OpCase(i_bs_init_fail_yjId): {
	 tmp_arg1 = yb(Arg(0));
	 tmp_arg2 = 0;
	 I++;
	 goto do_bs_init;
     }

     OpCase(i_bs_init_fail_xjId): {
	 tmp_arg1 = xb(Arg(0));
	 tmp_arg2 = 0;
	 I++;
     }
	 /* FALL THROUGH */
     do_bs_init:
         if (is_small(tmp_arg1)) {
	     Sint size = signed_val(tmp_arg1);
	     if (size < 0) {
		 goto badarg;
	     }
	     tmp_arg1 = (Eterm) size;
	 } else {
	     Uint bytes;

	     if (!term_to_Uint(tmp_arg1, &bytes)) {
		 c_p->freason = bytes;
		 goto lb_Cl_error;
	     }
	     if ((bytes >> (8*sizeof(Uint)-3)) != 0) {
		 goto system_limit;
	     }
	     tmp_arg1 = (Eterm) bytes;
	 }
	 if (tmp_arg1 <= ERL_ONHEAP_BIN_LIMIT) {
	     goto do_heap_bin_alloc;
	 } else {
	     goto do_proc_bin_alloc;
	 }


     OpCase(i_bs_init_heap_IIId): {
	 tmp_arg1 = Arg(0);
	 tmp_arg2 = Arg(1);
	 I++;
	 goto do_proc_bin_alloc;
     }

     OpCase(i_bs_init_IId): {
	 tmp_arg1 = Arg(0);
	 tmp_arg2 = 0;
     }
     /* FALL THROUGH */
     do_proc_bin_alloc: {
	 Binary* bptr;
	 ProcBin* pb;

	 erts_bin_offset = 0;
	 erts_writable_bin = 0;
	 TestBinVHeap(tmp_arg1 / sizeof(Eterm), 
	 	      tmp_arg2 + PROC_BIN_SIZE + ERL_SUB_BIN_SIZE, Arg(1));

	 /*
	  * Allocate the binary struct itself.
	  */
	 bptr = erts_bin_nrml_alloc(tmp_arg1);
	 erts_refc_init(&bptr->refc, 1);
	 erts_current_bin = (byte *) bptr->orig_bytes;

	 /*
	  * Now allocate the ProcBin on the heap.
	  */
	 pb = (ProcBin *) HTOP;
	 HTOP += PROC_BIN_SIZE;
	 pb->thing_word = HEADER_PROC_BIN;
	 pb->size = tmp_arg1;
	 pb->next = MSO(c_p).first;
	 MSO(c_p).first = (struct erl_off_heap_header*) pb;
	 pb->val = bptr;
	 pb->bytes = (byte*) bptr->orig_bytes;
	 pb->flags = 0;
	 
	 OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm));

	 StoreBifResult(2, make_binary(pb));
     }

     OpCase(i_bs_init_heap_bin_heap_IIId): {
	 tmp_arg1 = Arg(0);
	 tmp_arg2 = Arg(1);
	 I++;
	 goto do_heap_bin_alloc;
     }

     OpCase(i_bs_init_heap_bin_IId): {
	 tmp_arg1 = Arg(0);
	 tmp_arg2 = 0;
     }
     /* Fall through */
     do_heap_bin_alloc:
	 {
	     ErlHeapBin* hb;
	     Uint bin_need;

	     bin_need = heap_bin_size(tmp_arg1);
	     erts_bin_offset = 0;
	     erts_writable_bin = 0;
	     TestHeap(bin_need+tmp_arg2+ERL_SUB_BIN_SIZE, Arg(1));
	     hb = (ErlHeapBin *) HTOP;
	     HTOP += bin_need;
	     hb->thing_word = header_heap_bin(tmp_arg1);
	     hb->size = tmp_arg1;
	     erts_current_bin = (byte *) hb->data;
	     tmp_arg1 = make_binary(hb);
	     StoreBifResult(2, tmp_arg1);
	 }
 }

 OpCase(i_bs_add_jId): {
     Uint Unit = Arg(1);
     if (is_both_small(tmp_arg1, tmp_arg2)) {
	 Sint Arg1 = signed_val(tmp_arg1);
	 Sint Arg2 = signed_val(tmp_arg2);

	 if (Arg1 >= 0 && Arg2 >= 0) {
	     BsSafeMul(Arg2, Unit, goto system_limit, tmp_arg1);
	     tmp_arg1 += Arg1;

	 store_bs_add_result:
	     if (tmp_arg1 <= MAX_SMALL) {
		 tmp_arg1 = make_small(tmp_arg1);
	     } else {
		 /*
		  * May generate a heap fragment, but in this
		  * particular case it is OK, since the value will be
		  * stored into an x register (the GC will scan x
		  * registers for references to heap fragments) and
		  * there is no risk that value can be stored into a
		  * location that is not scanned for heap-fragment
		  * references (such as the heap).
		  */
		 SWAPOUT;
		 tmp_arg1 = erts_make_integer(tmp_arg1, c_p);
		 HTOP = HEAP_TOP(c_p);
	     }
	     StoreBifResult(2, tmp_arg1);
	 }
	 goto badarg;
     } else {
	 Uint a;
	 Uint b;
	 Uint c;

	 /*
	  * Now we know that one of the arguments is
	  * not a small. We must convert both arguments
	  * to Uints and check for errors at the same time.
	  *
	  * Error checking is tricky.
	  *
	  * If one of the arguments is not numeric or
	  * not positive, the error reason is BADARG.
	  *
	  * Otherwise if both arguments are numeric,
	  * but at least one argument does not fit in
	  * an Uint, the reason is SYSTEM_LIMIT.
	  */

	 if (!term_to_Uint(tmp_arg1, &a)) {
	     if (a == BADARG) {
		 goto badarg;
	     }
	     if (!term_to_Uint(tmp_arg2, &b)) {
		 c_p->freason = b;
		 goto lb_Cl_error;
	     }
	     goto system_limit;
	 } else if (!term_to_Uint(tmp_arg2, &b)) {
	     c_p->freason = b;
	     goto lb_Cl_error;
	 }

	 /*
	  * The arguments are now correct and stored in a and b.
	  */
	 
	 BsSafeMul(b, Unit, goto system_limit, c);
	 tmp_arg1 = a + c;
	 if (tmp_arg1 < a) {
	     /*
	      * If the result is less than one of the
	      * arguments, there must have been an overflow.
	      */
	     goto system_limit;
	 }
	 goto store_bs_add_result;
     }
     /* No fallthrough */
     ASSERT(0);
 }

 OpCase(bs_put_string_II):
    {
	BeamInstr *next;
	PreFetch(2, next);
	erts_new_bs_put_string(ERL_BITS_ARGS_2((byte *) Arg(1), Arg(0)));
	NextPF(2, next);
    }

 /*
  * tmp_arg1 = Number of bytes to build
  * tmp_arg2 = Source binary
  * Operands: Fail ExtraHeap Live Unit Dst
  */

 OpCase(i_bs_append_jIIId): {
     Uint live = Arg(2);
     Uint res;

     SWAPOUT;
     reg[0] = r(0);
     reg[live] = tmp_arg2;
     res = erts_bs_append(c_p, reg, live, tmp_arg1, Arg(1), Arg(3));
     r(0) = reg[0];
     SWAPIN;
     if (is_non_value(res)) {
	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
	 goto lb_Cl_error;
     }
     StoreBifResult(4, res);
 }

 /*
  * tmp_arg1 = Number of bytes to build
  * tmp_arg2 = Source binary
  * Operands: Fail Unit Dst
  */
 OpCase(i_bs_private_append_jId): {
     Eterm res;

     res = erts_bs_private_append(c_p, tmp_arg2, tmp_arg1, Arg(1));
     if (is_non_value(res)) {
	 /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
	 goto lb_Cl_error;
     }
     StoreBifResult(2, res);
 }

 /*
  * tmp_arg1 = Initial size of writable binary
  * Operands: Live Dst
  */
 OpCase(bs_init_writable): {
     SWAPOUT;
     r(0) = erts_bs_init_writable(c_p, r(0));
     SWAPIN;
     Next(0);
 }

 /*
  * Calculate the number of bytes needed to encode the source
  * operarand to UTF-8. If the source operand is invalid (e.g. wrong
  * type or range) we return a nonsense integer result (0 or 4). We
  * can get away with that because we KNOW that bs_put_utf8 will do
  * full error checking.
  */
 OpCase(i_bs_utf8_size_sd): {
     Eterm arg;
     Eterm result;

     GetArg1(0, arg);
     if (arg < make_small(0x80UL)) {
	 result = make_small(1);
     } else if (arg < make_small(0x800UL)) {
	 result = make_small(2);
     } else if (arg < make_small(0x10000UL)) {
	 result = make_small(3);
     } else {
	 result = make_small(4);
     }
     StoreBifResult(1, result);
 }

 OpCase(i_bs_put_utf8_js): {
     Eterm arg;

     GetArg1(1, arg);
     if (!erts_bs_put_utf8(ERL_BITS_ARGS_1(arg))) {
	 goto badarg;
     }
     Next(2);
 }

 /*
  * Calculate the number of bytes needed to encode the source
  * operarand to UTF-8. If the source operand is invalid (e.g. wrong
  * type or range) we return a nonsense integer result (2 or 4). We
  * can get away with that because we KNOW that bs_put_utf16 will do
  * full error checking.
  */

 OpCase(i_bs_utf16_size_sd): {
     Eterm arg;
     Eterm result = make_small(2);

     GetArg1(0, arg);
     if (arg >= make_small(0x10000UL)) {
	 result = make_small(4);
     }
     StoreBifResult(1, result);
 }

 OpCase(i_bs_put_utf16_jIs): {
     Eterm arg;

     GetArg1(2, arg);
     if (!erts_bs_put_utf16(ERL_BITS_ARGS_2(arg, Arg(1)))) {
	 goto badarg;
     }
     Next(3);
 }

 /*
  * Only used for validating a value about to be stored in a binary.
  */
 OpCase(i_bs_validate_unicode_js): {
     Eterm val;

     GetArg1(1, val);

     /*
      * There is no need to untag the integer, but it IS necessary
      * to make sure it is small (if the term is a bignum, it could
      * slip through the test, and there is no further test that
      * would catch it, since bit syntax construction silently masks
      * too big numbers).
      */
     if (is_not_small(val) || val > make_small(0x10FFFFUL) ||
	 (make_small(0xD800UL) <= val && val <= make_small(0xDFFFUL))) {
	 goto badarg;
     }
     Next(2);
 }

 /*
  * Only used for validating a value matched out. 
  *
  * tmp_arg1 = Integer to validate
  * tmp_arg2 = Match context
  */
 OpCase(i_bs_validate_unicode_retract_j): {
     /*
      * There is no need to untag the integer, but it IS necessary
      * to make sure it is small (a bignum pointer could fall in
      * the valid range).
      */
     if (is_not_small(tmp_arg1) || tmp_arg1 > make_small(0x10FFFFUL) ||
	 (make_small(0xD800UL) <= tmp_arg1 &&
	  tmp_arg1 <= make_small(0xDFFFUL))) {
	 ErlBinMatchBuffer *mb = ms_matchbuffer(tmp_arg2);

	 mb->offset -= 32;
	 goto badarg;
     }
     Next(1);
 }

 /*
  * Matching of binaries.
  */

 {
     Eterm header;
     BeamInstr *next;
     Uint slots;
     Eterm context;

     OpCase(i_bs_start_match2_rfIId): {
	 context = r(0);

     do_start_match:
	 slots = Arg(2);
	 if (!is_boxed(context)) {
	     ClauseFail();
	 }
	 PreFetch(4, next);
	 header = *boxed_val(context);
	 if (header_is_bin_matchstate(header)) {
	     ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context);
	     Uint actual_slots = HEADER_NUM_SLOTS(header);
	     ms->save_offset[0] = ms->mb.offset;
	     if (actual_slots < slots) {
		 ErlBinMatchState* dst;
		 Uint live = Arg(1);
		 Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);

		 TestHeapPreserve(wordsneeded, live, context);
		 ms = (ErlBinMatchState *) boxed_val(context);
		 dst = (ErlBinMatchState *) HTOP;
		 *dst = *ms;
		 *HTOP = HEADER_BIN_MATCHSTATE(slots);
		 HTOP += wordsneeded;
		 HEAP_SPACE_VERIFIED(0);
		 StoreResult(make_matchstate(dst), Arg(3));
	     }
	 } else if (is_binary_header(header)) {
	     Eterm result;
	     Uint live = Arg(1);
	     Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots);
	     TestHeapPreserve(wordsneeded, live, context);
	     HEAP_TOP(c_p) = HTOP;
#ifdef DEBUG
	     c_p->stop = E;	/* Needed for checking in HeapOnlyAlloc(). */
#endif
# 4357 "beam/beam_emu.c"
	     result = erts_bs_start_match_2(c_p, context, slots);
	     HTOP = HEAP_TOP(c_p);
	     HEAP_SPACE_VERIFIED(0);
	     if (is_non_value(result)) {
		 ClauseFail();
	     } else {
		 StoreResult(result, Arg(3));
	     }
	 } else {
	     ClauseFail();
	 }
	 NextPF(4, next);
     }
     OpCase(i_bs_start_match2_xfIId): {
	 context = xb(Arg(0));
	 I++;
	 goto do_start_match;
     }
     OpCase(i_bs_start_match2_yfIId): {
	 context = yb(Arg(0));
	 I++;
	 goto do_start_match;
     }
 }

 OpCase(bs_test_zero_tail2_fr): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     
     PreFetch(1, next);
     _mb = (ErlBinMatchBuffer*) ms_matchbuffer(r(0));
     if (_mb->size != _mb->offset) {
	 ClauseFail();
     }
     NextPF(1, next);
 }

 OpCase(bs_test_zero_tail2_fx): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     
     PreFetch(2, next);
     _mb = (ErlBinMatchBuffer*) ms_matchbuffer(xb(Arg(1)));
     if (_mb->size != _mb->offset) {
	 ClauseFail();
     }
     NextPF(2, next);
 }

 OpCase(bs_test_tail_imm2_frI): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     PreFetch(2, next);
     _mb = ms_matchbuffer(r(0));
     if (_mb->size - _mb->offset != Arg(1)) {
	 ClauseFail();
     }
     NextPF(2, next);
 }
 OpCase(bs_test_tail_imm2_fxI): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     PreFetch(3, next);
     _mb = ms_matchbuffer(xb(Arg(1)));
     if (_mb->size - _mb->offset != Arg(2)) {
	 ClauseFail();
     }
     NextPF(3, next);
 }

 OpCase(bs_test_unit_frI): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     PreFetch(2, next);
     _mb = ms_matchbuffer(r(0));
     if ((_mb->size - _mb->offset) % Arg(1)) {
	 ClauseFail();
     }
     NextPF(2, next);
 }
 OpCase(bs_test_unit_fxI): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     PreFetch(3, next);
     _mb = ms_matchbuffer(xb(Arg(1)));
     if ((_mb->size - _mb->offset) % Arg(2)) {
	 ClauseFail();
     }
     NextPF(3, next);
 }

 OpCase(bs_test_unit8_fr): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     PreFetch(1, next);
     _mb = ms_matchbuffer(r(0));
     if ((_mb->size - _mb->offset) & 7) {
	 ClauseFail();
     }
     NextPF(1, next);
 }
 OpCase(bs_test_unit8_fx): {
     BeamInstr *next;
     ErlBinMatchBuffer *_mb;
     PreFetch(2, next);
     _mb = ms_matchbuffer(xb(Arg(1)));
     if ((_mb->size - _mb->offset) & 7) {
	 ClauseFail();
     }
     NextPF(2, next);
 }

 {
     Eterm bs_get_integer8_context;

 OpCase(i_bs_get_integer_8_rfd): {
	 bs_get_integer8_context = r(0);
	 goto do_bs_get_integer_8;
     }

 OpCase(i_bs_get_integer_8_xfd): {
	 bs_get_integer8_context = xb(Arg(0));
	 I++;
     }

 do_bs_get_integer_8: {
	 ErlBinMatchBuffer *_mb;
	 Eterm _result;
	 _mb = ms_matchbuffer(bs_get_integer8_context);
	 if (_mb->size - _mb->offset < 8) {
	     ClauseFail();
	 }
	 if (BIT_OFFSET(_mb->offset) != 0) {
	     _result = erts_bs_get_integer_2(c_p, 8, 0, _mb);
	 } else {
	     _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]);
	     _mb->offset += 8;
	 }
	 StoreBifResult(1, _result);
     }
 }

 {
     Eterm bs_get_integer_16_context;

 OpCase(i_bs_get_integer_16_rfd):
     bs_get_integer_16_context = r(0);
     goto do_bs_get_integer_16;

 OpCase(i_bs_get_integer_16_xfd):
     bs_get_integer_16_context = xb(Arg(0));
     I++;

 do_bs_get_integer_16:
     {
	 ErlBinMatchBuffer *_mb;
	 Eterm _result;
	 _mb = ms_matchbuffer(bs_get_integer_16_context);
	 if (_mb->size - _mb->offset < 16) {
	     ClauseFail(); 
	 }
	 if (BIT_OFFSET(_mb->offset) != 0) {
	     _result = erts_bs_get_integer_2(c_p, 16, 0, _mb);
	 } else {
	     _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset)));
	     _mb->offset += 16;
	 }
	 StoreBifResult(1, _result);
     }
 }

 {
     Eterm bs_get_integer_32_context;

 OpCase(i_bs_get_integer_32_rfId):
     bs_get_integer_32_context = r(0);
     goto do_bs_get_integer_32;

     
 OpCase(i_bs_get_integer_32_xfId):
     bs_get_integer_32_context = xb(Arg(0));
     I++;


 do_bs_get_integer_32:
     {
	 ErlBinMatchBuffer *_mb;
	 Uint32 _integer;
	 Eterm _result;
	 _mb = ms_matchbuffer(bs_get_integer_32_context);
	 if (_mb->size - _mb->offset < 32) { ClauseFail(); }
	 if (BIT_OFFSET(_mb->offset) != 0) {
	     _integer = erts_bs_get_unaligned_uint32(_mb);
	 } else {
	     _integer = get_int32(_mb->base + _mb->offset/8);
	 }
	 _mb->offset += 32;
#if !defined(ARCH_64) || HALFWORD_HEAP
	 if (IS_USMALL(0, _integer)) {
#endif
# 4557 "beam/beam_emu.c"
	     _result = make_small(_integer);
#if !defined(ARCH_64) || HALFWORD_HEAP
	 } else {
	     TestHeap(BIG_UINT_HEAP_SIZE, Arg(1));
	     _result = uint_to_big((Uint) _integer, HTOP);
	     HTOP += BIG_UINT_HEAP_SIZE;
	     HEAP_SPACE_VERIFIED(0);
	 }
#endif
# 4566 "beam/beam_emu.c"
	 StoreBifResult(2, _result);
     }
 }

 /* Operands: Size Live Fail Flags Dst */
 OpCase(i_bs_get_integer_imm_rIIfId): {
     tmp_arg1 = r(0);
     /* Operands: Size Live Fail Flags Dst */
     goto do_bs_get_integer_imm_test_heap;
 }

 /* Operands: x(Reg) Size Live Fail Flags Dst */
 OpCase(i_bs_get_integer_imm_xIIfId): {
     tmp_arg1 = xb(Arg(0));
     I++;
     /* Operands: Size Live Fail Flags Dst */
     goto do_bs_get_integer_imm_test_heap;
 }

 /*
  * tmp_arg1 = match context
  * Operands: Size Live Fail Flags Dst
  */
 do_bs_get_integer_imm_test_heap: {
     Uint wordsneeded;
     tmp_arg2 = Arg(0);
     wordsneeded = 1+WSIZE(NBYTES(tmp_arg2));
     TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
     I += 2;
     /* Operands: Fail Flags Dst */
     goto do_bs_get_integer_imm;
 }

 /* Operands: Size Fail Flags Dst */
 OpCase(i_bs_get_integer_small_imm_rIfId): {
     tmp_arg1 = r(0);
     tmp_arg2 = Arg(0);
     I++;
     /* Operands: Fail Flags Dst */
     goto do_bs_get_integer_imm;
 }

 /* Operands: x(Reg) Size Fail Flags Dst */
 OpCase(i_bs_get_integer_small_imm_xIfId): {
     tmp_arg1 = xb(Arg(0));
     tmp_arg2 = Arg(1);
     I += 2;
     /* Operands: Fail Flags Dst */
     goto do_bs_get_integer_imm;
 }
 
 /*
  * tmp_arg1 = match context
  * tmp_arg2 = size of field
  * Operands: Fail Flags Dst
  */
 do_bs_get_integer_imm: {
     ErlBinMatchBuffer* mb;
     Eterm result;

     mb = ms_matchbuffer(tmp_arg1);
     LIGHT_SWAPOUT;
     result = erts_bs_get_integer_2(c_p, tmp_arg2, Arg(1), mb);
     LIGHT_SWAPIN;
     HEAP_SPACE_VERIFIED(0);
     if (is_non_value(result)) {
	 ClauseFail();
     }
     StoreBifResult(2, result);
 }

 /*
  * tmp_arg1 = Match context
  * tmp_arg2 = Size field
  * Operands: Fail Live FlagsAndUnit Dst
  */
 OpCase(i_bs_get_integer_fIId): {
     Uint flags;
     Uint size;
     ErlBinMatchBuffer* mb;
     Eterm result;

     flags = Arg(2);
     BsGetFieldSize(tmp_arg2, (flags >> 3), ClauseFail(), size);
     if (size >= SMALL_BITS) {
	 Uint wordsneeded;
	 /* check bits size before potential gc.
	  * We do not want a gc and then realize we don't need
	  * the allocated space (i.e. if the op fails)
	  *
	  * remember to reacquire the matchbuffer after gc.
	  */

	 mb = ms_matchbuffer(tmp_arg1);
	 if (mb->size - mb->offset < size) {
	     ClauseFail();
	 }
	 wordsneeded = 1+WSIZE(NBYTES((Uint) size));
	 TestHeapPreserve(wordsneeded, Arg(1), tmp_arg1);
     }
     mb = ms_matchbuffer(tmp_arg1);
     LIGHT_SWAPOUT;
     result = erts_bs_get_integer_2(c_p, size, flags, mb);
     LIGHT_SWAPIN;
     HEAP_SPACE_VERIFIED(0);
     if (is_non_value(result)) {
	 ClauseFail();
     }
     StoreBifResult(3, result);
 }

 {
     Eterm get_utf8_context;

     /* Operands: MatchContext Fail Dst */
 OpCase(i_bs_get_utf8_rfd): {
	 get_utf8_context = r(0);
	 goto do_bs_get_utf8;
     }

 OpCase(i_bs_get_utf8_xfd): {
	 get_utf8_context = xb(Arg(0));
	 I++;
     }

     /*
      * get_utf8_context = match_context
      * Operands: Fail Dst
      */

 do_bs_get_utf8: {
	 Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context));
	 if (is_non_value(result)) {
	     ClauseFail();
	 }
	 StoreBifResult(1, result);
     }
 }

 {
     Eterm get_utf16_context;

     /* Operands: MatchContext Fail Flags Dst */
 OpCase(i_bs_get_utf16_rfId): {
	 get_utf16_context = r(0);
	 goto do_bs_get_utf16;
     }

 OpCase(i_bs_get_utf16_xfId): {
	 get_utf16_context = xb(Arg(0));
	 I++;
     }

     /*
      * get_utf16_context = match_context
      * Operands: Fail Flags Dst
      */
 do_bs_get_utf16: {
	 Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context),
					  Arg(1));
	 if (is_non_value(result)) {
	     ClauseFail();
	 }
	 StoreBifResult(2, result);
     }
 }

 {
     Eterm context_to_binary_context;
     ErlBinMatchBuffer* mb;
     ErlSubBin* sb;
     Uint size;
     Uint offs;
     Uint orig;
     Uint hole_size;

     OpCase(bs_context_to_binary_r): {
	 context_to_binary_context = x0;
	 I -= 2;
	 goto do_context_to_binary;
     }

     /* Unfortunately, inlining can generate this instruction. */
     OpCase(bs_context_to_binary_y): {
	 context_to_binary_context = yb(Arg(0));
	 goto do_context_to_binary0;
     }

     OpCase(bs_context_to_binary_x): {
	 context_to_binary_context = xb(Arg(0));
     
     do_context_to_binary0:
	 I--;
     }

 do_context_to_binary:
     if (is_boxed(context_to_binary_context) &&
	 header_is_bin_matchstate(*boxed_val(context_to_binary_context))) {
	 ErlBinMatchState* ms;
	 ms = (ErlBinMatchState *) boxed_val(context_to_binary_context);
	 mb = &ms->mb;
	 offs = ms->save_offset[0];
	 size = mb->size - offs;
	 goto do_bs_get_binary_all_reuse_common;
     }
     Next(2);

     OpCase(i_bs_get_binary_all_reuse_rfI): {
	 context_to_binary_context = x0;
	 goto do_bs_get_binary_all_reuse;
     }

     OpCase(i_bs_get_binary_all_reuse_xfI): {
	 context_to_binary_context = xb(Arg(0));
	 I++;
     }

 do_bs_get_binary_all_reuse:
     mb = ms_matchbuffer(context_to_binary_context);
     size = mb->size - mb->offset;
     if (size % Arg(1) != 0) {
	 ClauseFail();
     }
     offs = mb->offset;

 do_bs_get_binary_all_reuse_common:
     orig = mb->orig;
     sb = (ErlSubBin *) boxed_val(context_to_binary_context);
     hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE;
     sb->thing_word = HEADER_SUB_BIN;
     sb->size = BYTE_OFFSET(size);
     sb->bitsize = BIT_OFFSET(size);
     sb->offs = BYTE_OFFSET(offs);
     sb->bitoffs = BIT_OFFSET(offs);
     sb->is_writable = 0;
     sb->orig = orig;
     if (hole_size) {
	 sb[1].thing_word = make_pos_bignum_header(hole_size-1);
     }
     Next(2);
 }

 {
     Eterm match_string_context;

     OpCase(i_bs_match_string_rfII): {
	 match_string_context = r(0);
	 goto do_bs_match_string;
     }
     OpCase(i_bs_match_string_xfII): {
	 match_string_context = xb(Arg(0));
	 I++;
     }

 do_bs_match_string:
     {
	 BeamInstr *next;
	 byte* bytes;
	 Uint bits;
	 ErlBinMatchBuffer* mb;
	 Uint offs;

	 PreFetch(3, next);
	 bits = Arg(1);
	 bytes = (byte *) Arg(2);
	 mb = ms_matchbuffer(match_string_context);
	 if (mb->size - mb->offset < bits) {
	     ClauseFail();
	 }
	 offs = mb->offset & 7;
	 if (offs == 0 && (bits & 7) == 0) {
	     if (sys_memcmp(bytes, mb->base+(mb->offset>>3), bits>>3)) {
		 ClauseFail();
	     }
	 } else if (erts_cmp_bits(bytes, 0, mb->base+(mb->offset>>3), mb->offset & 7, bits)) {
	     ClauseFail();
	 }
	 mb->offset += bits;
	 NextPF(3, next);
     }
 }

 OpCase(i_bs_save2_rI): {
     BeamInstr *next;
     ErlBinMatchState *_ms;
     PreFetch(1, next);
     _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
     _ms->save_offset[Arg(0)] = _ms->mb.offset;
     NextPF(1, next);
 }
 OpCase(i_bs_save2_xI): {
     BeamInstr *next;
     ErlBinMatchState *_ms;
     PreFetch(2, next);
     _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
     _ms->save_offset[Arg(1)] = _ms->mb.offset;
     NextPF(2, next);
 }

 OpCase(i_bs_restore2_rI): {
     BeamInstr *next;
     ErlBinMatchState *_ms;
     PreFetch(1, next);
     _ms = (ErlBinMatchState*) boxed_val((Eterm) r(0));
     _ms->mb.offset = _ms->save_offset[Arg(0)];
     NextPF(1, next);
 }
 OpCase(i_bs_restore2_xI): {
     BeamInstr *next;
     ErlBinMatchState *_ms;
     PreFetch(2, next);
     _ms = (ErlBinMatchState*) boxed_val((Eterm) xb(Arg(0)));
     _ms->mb.offset = _ms->save_offset[Arg(1)];
     NextPF(2, next);
 }

#if 0 /* expanded by -frewrite-includes */
#include "beam_cold.h"
#endif /* expanded by -frewrite-includes */
# 4882 "beam/beam_emu.c"
# 1 "armv6-portbld-freebsd11.0/opt/smp/beam_cold.h" 1
/*
 *  Warning: Do not edit this file.
 *  Auto-generated by 'beam_makeops'.
 */

OpCase(i_bs_get_binary2_frIsId):
    { Eterm targ1; Eterm dst; 
    BeamInstr* next;
    PreFetch(5, next);
    GetR(2, targ1);
    dst = Arg(4);
    BsGetBinary_2(r(0), Arg(1), targ1, Arg(3), dst, StoreResult, ClauseFail());
    NextPF(5, next);
    }

OpCase(i_bs_get_binary2_fxIsId):
    { Eterm targ1; Eterm dst; 
    BeamInstr* next;
    PreFetch(6, next);
    GetR(3, targ1);
    dst = Arg(5);
    BsGetBinary_2(xb(Arg(1)), Arg(2), targ1, Arg(4), dst, StoreResult, ClauseFail());
    NextPF(6, next);
    }

OpCase(i_bs_get_binary_all2_frIId):
    { Eterm dst; 
    BeamInstr* next;
    PreFetch(4, next);
    dst = Arg(3);
    BsGetBinaryAll_2(r(0), Arg(1), Arg(2), dst, StoreResult, ClauseFail());
    NextPF(4, next);
    }

OpCase(i_bs_get_binary_all2_fxIId):
    { Eterm dst; 
    BeamInstr* next;
    PreFetch(5, next);
    dst = Arg(4);
    BsGetBinaryAll_2(xb(Arg(1)), Arg(2), Arg(3), dst, StoreResult, ClauseFail());
    NextPF(5, next);
    }

OpCase(i_bs_get_binary_imm2_frIIId):
    { Eterm dst; 
    BeamInstr* next;
    PreFetch(5, next);
    dst = Arg(4);
    BsGetBinaryImm_2(r(0), Arg(1), Arg(2), Arg(3), dst, StoreResult, ClauseFail());
    NextPF(5, next);
    }

OpCase(i_bs_get_binary_imm2_fxIIId):
    { Eterm dst; 
    BeamInstr* next;
    PreFetch(6, next);
    dst = Arg(5);
    BsGetBinaryImm_2(xb(Arg(1)), Arg(2), Arg(3), Arg(4), dst, StoreResult, ClauseFail());
    NextPF(6, next);
    }

OpCase(i_bs_get_float2_frIsId):
    { Eterm targ1; Eterm dst; 
    BeamInstr* next;
    PreFetch(5, next);
    GetR(2, targ1);
    dst = Arg(4);
    BsGetFloat2(r(0), Arg(1), targ1, Arg(3), dst, StoreResult, ClauseFail());
    NextPF(5, next);
    }

OpCase(i_bs_get_float2_fxIsId):
    { Eterm targ1; Eterm dst; 
    BeamInstr* next;
    PreFetch(6, next);
    GetR(3, targ1);
    dst = Arg(5);
    BsGetFloat2(xb(Arg(1)), Arg(2), targ1, Arg(4), dst, StoreResult, ClauseFail());
    NextPF(6, next);
    }

OpCase(i_bs_skip_bits2_frxI):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    BsSkipBits2(r(0), xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_bs_skip_bits2_fryI):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    BsSkipBits2(r(0), yb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_bs_skip_bits2_fxrI):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    BsSkipBits2(xb(Arg(1)), r(0), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_bs_skip_bits2_fxxI):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    BsSkipBits2(xb(Arg(1)), xb(Arg(2)), Arg(3), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_bs_skip_bits2_fxyI):
    { 
    BeamInstr* next;
    PreFetch(4, next);
    BsSkipBits2(xb(Arg(1)), yb(Arg(2)), Arg(3), ClauseFail());
    NextPF(4, next);
    }

OpCase(i_bs_skip_bits_all2_frI):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    BsSkipBitsAll2(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_bs_skip_bits_all2_fxI):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    BsSkipBitsAll2(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_bs_skip_bits_imm2_frI):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    BsSkipBitsImm2(r(0), Arg(1), ClauseFail());
    NextPF(2, next);
    }

OpCase(i_bs_skip_bits_imm2_fxI):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    BsSkipBitsImm2(xb(Arg(1)), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_fetch_cc):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    FetchArgs(Arg(0), Arg(1));
    NextPF(2, next);
    }

OpCase(i_fetch_ss):
    { Eterm targ1; Eterm targ2; 
    BeamInstr* next;
    PreFetch(2, next);
    GetR(0, targ1);
    GetR(1, targ2);
    FetchArgs(targ1, targ2);
    NextPF(2, next);
    }

OpCase(i_get_tuple_element_rPr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    GetTupleElement(r(0), Arg(0), r(0));
    NextPF(1, next);
    }

OpCase(i_get_tuple_element_rPy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    GetTupleElement(r(0), Arg(0), yb(Arg(1)));
    NextPF(2, next);
    }

OpCase(i_get_tuple_element_xPy):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetTupleElement(xb(Arg(0)), Arg(1), yb(Arg(2)));
    NextPF(3, next);
    }

OpCase(i_get_tuple_element_yPy):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    GetTupleElement(yb(Arg(0)), Arg(1), yb(Arg(2)));
    NextPF(3, next);
    }

OpCase(i_is_eq_exact_spec_frr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    EqualExact(r(0), r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(i_is_ge_spec_fcc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsGreaterEqual(Arg(1), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_lt_spec_fcc):
    { 
    BeamInstr* next;
    PreFetch(3, next);
    IsLessThan(Arg(1), Arg(2), ClauseFail());
    NextPF(3, next);
    }

OpCase(i_is_lt_spec_frr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsLessThan(r(0), r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(i_make_fun_It):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    MakeFun(Arg(0), Arg(1));
    NextPF(2, next);
    }

OpCase(i_new_bs_put_binary_all_jsI):
    { Eterm targ1; 
    BeamInstr* next;
    PreFetch(3, next);
    GetR(1, targ1);
    NewBsPutBinaryAll(targ1, Arg(2));
    NextPF(3, next);
    }

OpCase(i_new_bs_put_binary_imm_jIs):
    { Eterm targ1; 
    BeamInstr* next;
    PreFetch(3, next);
    GetR(2, targ1);
    NewBsPutBinaryImm(Arg(1), targ1);
    NextPF(3, next);
    }

OpCase(i_new_bs_put_binary_jsIs):
    { Eterm targ1; Eterm targ2; 
    BeamInstr* next;
    PreFetch(4, next);
    GetR(1, targ1);
    GetR(3, targ2);
    NewBsPutBinary(targ1, Arg(2), targ2);
    NextPF(4, next);
    }

OpCase(i_new_bs_put_float_imm_jIIs):
    { Eterm targ1; 
    BeamInstr* next;
    PreFetch(4, next);
    GetR(3, targ1);
    NewBsPutFloatImm(Arg(1), Arg(2), targ1);
    NextPF(4, next);
    }

OpCase(i_new_bs_put_float_jsIs):
    { Eterm targ1; Eterm targ2; 
    BeamInstr* next;
    PreFetch(4, next);
    GetR(1, targ1);
    GetR(3, targ2);
    NewBsPutFloat(targ1, Arg(2), targ2);
    NextPF(4, next);
    }

OpCase(i_new_bs_put_integer_imm_jIIs):
    { Eterm targ1; 
    BeamInstr* next;
    PreFetch(4, next);
    GetR(3, targ1);
    NewBsPutIntegerImm(Arg(1), Arg(2), targ1);
    NextPF(4, next);
    }

OpCase(i_new_bs_put_integer_jsIs):
    { Eterm targ1; Eterm targ2; 
    BeamInstr* next;
    PreFetch(4, next);
    GetR(1, targ1);
    GetR(3, targ2);
    NewBsPutInteger(targ1, Arg(2), targ2);
    NextPF(4, next);
    }

OpCase(is_atom_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsAtom(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_binary_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsBinary(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_bitstring_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsBitstring(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_boolean_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsBoolean(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_boolean_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsBoolean(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_boolean_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsBoolean(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_float_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsFloat(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_list_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsList(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_number_fr):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    IsNumber(r(0), ClauseFail());
    NextPF(1, next);
    }

OpCase(is_number_fx):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsNumber(xb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_number_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsNumber(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_pid_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsPid(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_port_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsPort(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(is_reference_fy):
    { 
    BeamInstr* next;
    PreFetch(2, next);
    IsRef(yb(Arg(1)), ClauseFail());
    NextPF(2, next);
    }

OpCase(node_y):
    { 
    BeamInstr* next;
    PreFetch(1, next);
    Node(yb(Arg(0)));
    NextPF(1, next);
    }

OpCase(put_list_ssd):
    { Eterm targ1; Eterm targ2; Eterm dst; 
    BeamInstr* next;
    PreFetch(3, next);
    GetR(0, targ1);
    GetR(1, targ2);
    dst = Arg(2);
    PutList(targ1, targ2, dst, StoreResult);
    NextPF(3, next);
    }

# 4883 "beam/beam_emu.c" 2


 /*
  * This instruction is probably never used (because it is combined with a
  * a return). However, a future compiler might for some reason emit a
  * deallocate not followed by a return, and that should work.
  */
 OpCase(deallocate_I): {
     BeamInstr *next;

     PreFetch(1, next);
     D(Arg(0));
     NextPF(1, next);
 }

    /*
     * Trace and debugging support.
     */

 OpCase(return_trace): {
     BeamInstr* code = (BeamInstr *) (UWord) E[0];
     
     SWAPOUT;		/* Needed for shared heap */
     ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
     erts_trace_return(c_p, code, r(0), E+1/*Process tracer*/);
     ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
     SWAPIN;
     c_p->cp = NULL;
     SET_I((BeamInstr *) cp_val(E[2]));
     E += 3;
     Goto(*I);
 }

 OpCase(i_generic_breakpoint): {
     BeamInstr real_I;
     ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI));
     SWAPOUT;
     reg[0] = r(0);
     real_I = erts_generic_breakpoint(c_p, I, reg);
     r(0) = reg[0];
     SWAPIN;
     ASSERT(VALID_INSTR(real_I));
     Goto(real_I);
 }

 OpCase(i_return_time_trace): {
     BeamInstr *pc = (BeamInstr *) (UWord) E[0];
     SWAPOUT;
     erts_trace_time_return(c_p, pc);
     SWAPIN;
     c_p->cp = NULL;
     SET_I((BeamInstr *) cp_val(E[1]));
     E += 2;
     Goto(*I);
 }

 OpCase(i_return_to_trace): {
     if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO)) {
	 Uint *cpp = (Uint*) E;
	 for(;;) {
	     ASSERT(is_CP(*cpp));
	     if (*cp_val(*cpp) == (BeamInstr) OpCode(return_trace)) {
		 do ++cpp; while(is_not_CP(*cpp));
		 cpp += 2;
	     } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) {
		 do ++cpp; while(is_not_CP(*cpp));
	     } else break;
	 }
	 SWAPOUT;		/* Needed for shared heap */
	 ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
	 erts_trace_return_to(c_p, cp_val(*cpp));
	 ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
	 SWAPIN;
     }
     c_p->cp = NULL;
     SET_I((BeamInstr *) cp_val(E[0]));
     E += 1;
     Goto(*I);
 }

 /*
  * New floating point instructions.
  */

 OpCase(fmove_ql): {
     Eterm fr = Arg(1);
     BeamInstr *next;

     PreFetch(2, next);
     GET_DOUBLE(Arg(0), *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
     NextPF(2, next);
 }

 OpCase(fmove_dl): {
     Eterm targ1;
     Eterm fr = Arg(1);
     BeamInstr *next;

     PreFetch(2, next);
     GetR(0, targ1);
     /* Arg(0) == HEADER_FLONUM */
     GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
     NextPF(2, next);
 }

 OpCase(fmove_ld): {
     Eterm fr = Arg(0);
     Eterm dest = make_float(HTOP);

     PUT_DOUBLE(*(FloatDef*)ADD_BYTE_OFFSET(freg, fr), HTOP);
     HTOP += FLOAT_SIZE_OBJECT;
     StoreBifResult(1, dest);
 }

 OpCase(fconv_dl): {
     Eterm targ1;
     Eterm fr = Arg(1);
     BeamInstr *next;

     GetR(0, targ1);
     PreFetch(2, next);
     if (is_small(targ1)) {
	 fb(fr) = (double) signed_val(targ1);
     } else if (is_big(targ1)) {
	 if (big_to_double(targ1, &fb(fr)) < 0) {
	     goto fbadarith;
	 }
     } else if (is_float(targ1)) {
	 GET_DOUBLE(targ1, *(FloatDef*)ADD_BYTE_OFFSET(freg, fr));
     } else {
	 goto fbadarith;
     }
     NextPF(2, next);
 }

#ifdef NO_FPE_SIGNALS
     OpCase(fclearerror):
     OpCase(i_fcheckerror):
	 erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)");
#  define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT
#  define ERTS_NO_FPE_ERROR ERTS_FP_ERROR
#else
# 5025 "beam/beam_emu.c"
#  define ERTS_NO_FPE_CHECK_INIT(p)
#  define ERTS_NO_FPE_ERROR(p, a, b)

     OpCase(fclearerror): {
	 BeamInstr *next;

	 PreFetch(0, next);
	 ERTS_FP_CHECK_INIT(c_p);
	 NextPF(0, next);
     }

     OpCase(i_fcheckerror): {
	 BeamInstr *next;

	 PreFetch(0, next);
	 ERTS_FP_ERROR(c_p, freg[0].fd, goto fbadarith);
	 NextPF(0, next);
     }
#endif
# 5044 "beam/beam_emu.c"


 OpCase(i_fadd_lll): {
     BeamInstr *next;

     PreFetch(3, next);
     ERTS_NO_FPE_CHECK_INIT(c_p);
     fb(Arg(2)) = fb(Arg(0)) + fb(Arg(1));
     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
     NextPF(3, next);
 }
 OpCase(i_fsub_lll): {
     BeamInstr *next;

     PreFetch(3, next);
     ERTS_NO_FPE_CHECK_INIT(c_p);
     fb(Arg(2)) = fb(Arg(0)) - fb(Arg(1));
     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
     NextPF(3, next);
 }
 OpCase(i_fmul_lll): {
     BeamInstr *next;

     PreFetch(3, next);
     ERTS_NO_FPE_CHECK_INIT(c_p);
     fb(Arg(2)) = fb(Arg(0)) * fb(Arg(1));
     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
     NextPF(3, next);
 }
 OpCase(i_fdiv_lll): {
     BeamInstr *next;

     PreFetch(3, next);
     ERTS_NO_FPE_CHECK_INIT(c_p);
     fb(Arg(2)) = fb(Arg(0)) / fb(Arg(1));
     ERTS_NO_FPE_ERROR(c_p, fb(Arg(2)), goto fbadarith);
     NextPF(3, next);
 }
 OpCase(i_fnegate_ll): {
     BeamInstr *next;

     PreFetch(2, next);
     ERTS_NO_FPE_CHECK_INIT(c_p);
     fb(Arg(1)) = -fb(Arg(0));
     ERTS_NO_FPE_ERROR(c_p, fb(Arg(1)), goto fbadarith);
     NextPF(2, next);

 fbadarith:
     c_p->freason = BADARITH;
     goto find_func_info;
 }

#ifdef HIPE
 {
     unsigned cmd;

     OpCase(hipe_trap_call): {
	 /*
	  * I[-5]: &&lb_i_func_info_IaaI
	  * I[-4]: Native code callee (inserted by HiPE)
	  * I[-3]: Module (tagged atom)
	  * I[-2]: Function (tagged atom)
	  * I[-1]: Arity (untagged integer)
	  * I[ 0]: &&lb_hipe_trap_call
	  * ... remainder of original BEAM code
	  */
	 ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
	 c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
	 cmd = HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8);
	 ++hipe_trap_count;
	 goto L_hipe_mode_switch;
     }
     OpCase(hipe_trap_call_closure): {
       ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
       c_p->hipe.u.ncallee = (void(*)(void)) I[-4];
       cmd = HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8);
       ++hipe_trap_count;
       goto L_hipe_mode_switch;
     }
     OpCase(hipe_trap_return): {
	 cmd = HIPE_MODE_SWITCH_CMD_RETURN;
	 goto L_hipe_mode_switch;
     }
     OpCase(hipe_trap_throw): {
	 cmd = HIPE_MODE_SWITCH_CMD_THROW;
	 goto L_hipe_mode_switch;
     }
     OpCase(hipe_trap_resume): {
	 cmd = HIPE_MODE_SWITCH_CMD_RESUME;
	 goto L_hipe_mode_switch;
     }
 L_hipe_mode_switch:
     /* XXX: this abuse of def_arg_reg[] is horrid! */
     SWAPOUT;
     c_p->fcalls = FCALLS;
     c_p->def_arg_reg[4] = -neg_o_reds;
     reg[0] = r(0);
     c_p = hipe_mode_switch(c_p, cmd, reg);
     reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array;
     freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array;
     ERL_BITS_RELOAD_STATEP(c_p);
     neg_o_reds = -c_p->def_arg_reg[4];
     FCALLS = c_p->fcalls;
     SWAPIN;
     switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */
       case HIPE_MODE_SWITCH_RES_RETURN:
	 ASSERT(is_value(reg[0]));
	 MoveReturn(reg[0], r(0));
       case HIPE_MODE_SWITCH_RES_CALL_EXPORTED:
	 c_p->i = c_p->hipe.u.callee_exp->addressv[erts_active_code_ix()];
	 /*fall through*/
       case HIPE_MODE_SWITCH_RES_CALL_BEAM:
	 SET_I(c_p->i);
	 r(0) = reg[0];
	 Dispatch();
       case HIPE_MODE_SWITCH_RES_CALL_CLOSURE:
	 /* This can be used to call any function value, but currently it's
	    only used to call closures referring to unloaded modules. */
	 {
	     BeamInstr *next;

	     next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
	     SWAPIN;
	     if (next != NULL) {
		 r(0) = reg[0];
		 SET_I(next);
		 Dispatchfun();
	     }
	     goto find_func_info;
	 }
       case HIPE_MODE_SWITCH_RES_THROW:
	 c_p->cp = NULL;
	 I = handle_error(c_p, I, reg, NULL);
	 goto post_error_handling;
       default:
	 erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]);
     }
 }
 OpCase(hipe_call_count): {
     /*
      * I[-5]: &&lb_i_func_info_IaaI
      * I[-4]: pointer to struct hipe_call_count (inserted by HiPE)
      * I[-3]: Module (tagged atom)
      * I[-2]: Function (tagged atom)
      * I[-1]: Arity (untagged integer)
      * I[ 0]: &&lb_hipe_call_count
      * ... remainder of original BEAM code
      */
     struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4];
     ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI));
     ASSERT(hcc != NULL);
     ASSERT(VALID_INSTR(hcc->opcode));
     ++(hcc->count);
     Goto(hcc->opcode);
 }
#endif /* HIPE */
# 5200 "beam/beam_emu.c"

 OpCase(i_yield):
 {
     /* This is safe as long as REDS_IN(c_p) is never stored 
      * in c_p->arg_reg[0]. It is currently stored in c_p->def_arg_reg[5],
      * which may be c_p->arg_reg[5], which is close, but no banana.
      */
     c_p->arg_reg[0] = am_true;
     c_p->arity = 1; /* One living register (the 'true' return value) */
     SWAPOUT;
     c_p->i = I + 1; /* Next instruction */
     c_p->current = NULL;
     goto do_schedule;
 }

 OpCase(i_hibernate): {
     SWAPOUT;
     if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
	 c_p->flags &= ~F_HIBERNATE_SCHED;
	 goto do_schedule;
     } else {
	 I = handle_error(c_p, I, reg, hibernate_3);
	 goto post_error_handling;
     }
 }

 OpCase(i_debug_breakpoint): {
     SWAPOUT;
     reg[0] = r(0);
     I = call_error_handler(c_p, I-3, reg, am_breakpoint);
     r(0) = reg[0];
     SWAPIN;
     if (I) {
	 Goto(*I);
     }
     goto handle_error;
 }


 OpCase(system_limit_j):
 system_limit:
    c_p->freason = SYSTEM_LIMIT;
    goto lb_Cl_error;


#ifdef ERTS_OPCODE_COUNTER_SUPPORT
    DEFINE_COUNTING_LABELS;
#endif
# 5248 "beam/beam_emu.c"

#ifndef NO_JUMP_TABLE
#ifdef DEBUG
 end_emulator_loop:
#endif
# 5253 "beam/beam_emu.c"
#endif
# 5254 "beam/beam_emu.c"

 OpCase(int_code_end):
 OpCase(label_L):
 OpCase(on_load):
 OpCase(line_I):
    erts_exit(ERTS_ERROR_EXIT, "meta op\n");

    /*
     * One-time initialization of Beam emulator.
     */

 init_emulator:
 {
     int i;
     Export* ep;

#ifndef NO_JUMP_TABLE
#ifdef ERTS_OPCODE_COUNTER_SUPPORT
#ifdef DEBUG
     counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
#endif
# 5275 "beam/beam_emu.c"
     counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
     beam_ops = counting_opcodes;
#else /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
# 5278 "beam/beam_emu.c"
     beam_ops = opcodes;
#endif /* ERTS_OPCODE_COUNTER_SUPPORT */
# 5280 "beam/beam_emu.c"
#endif /* NO_JUMP_TABLE */
# 5281 "beam/beam_emu.c"
     
     em_call_error_handler = OpCode(call_error_handler);
     em_apply_bif = OpCode(apply_bif);
     em_call_nif = OpCode(call_nif);

     beam_apply[0]             = (BeamInstr) OpCode(i_apply);
     beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
     beam_exit[0]              = (BeamInstr) OpCode(error_action_code);
     beam_continue_exit[0]     = (BeamInstr) OpCode(continue_exit);
     beam_return_to_trace[0]   = (BeamInstr) OpCode(i_return_to_trace);
     beam_return_trace[0]      = (BeamInstr) OpCode(return_trace);
     beam_exception_trace[0]   = (BeamInstr) OpCode(return_trace); /* UGLY */
     beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace);

     /*
      * Enter all BIFs into the export table.
      */
     for (i = 0; i < BIF_SIZE; i++) {
	 ep = erts_export_put(bif_table[i].module,
			      bif_table[i].name,
			      bif_table[i].arity);
	 bif_export[i] = ep;
	 ep->code[3] = (BeamInstr) OpCode(apply_bif);
	 ep->code[4] = (BeamInstr) bif_table[i].f;
	 /* XXX: set func info for bifs */
	 ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
     }

     return;
 }
#ifdef NO_JUMP_TABLE
 default:
    erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
  }
#endif
# 5316 "beam/beam_emu.c"
    return;			/* Never executed */

  save_calls1:
    {
	Eterm* dis_next;

	save_calls(c_p, (Export *) Arg(0));

	SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);

	dis_next = (Eterm *) *I;
	FCALLS--;
	Goto(dis_next);
    }
}

static BifFunction
translate_gc_bif(void* gcf)
{
    if (gcf == erts_gc_length_1) {
	return length_1;
    } else if (gcf == erts_gc_size_1) {
	return size_1;
    } else if (gcf == erts_gc_bit_size_1) {
	return bit_size_1;
    } else if (gcf == erts_gc_byte_size_1) {
	return byte_size_1;
    } else if (gcf == erts_gc_map_size_1) {
	return map_size_1;
    } else if (gcf == erts_gc_abs_1) {
	return abs_1;
    } else if (gcf == erts_gc_float_1) {
	return float_1;
    } else if (gcf == erts_gc_round_1) {
	return round_1;
    } else if (gcf == erts_gc_trunc_1) {
	return round_1;
    } else if (gcf == erts_gc_binary_part_2) {
	return binary_part_2;
    } else if (gcf == erts_gc_binary_part_3) {
	return binary_part_3;
    } else {
	erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
    }
}

/*
 * Mapping from the error code 'class tag' to atoms.
 */
Eterm exception_tag[NUMBER_EXC_TAGS] = {
  am_error,	/* 0 */
  am_exit,	/* 1 */
  am_throw,	/* 2 */
};

/*
 * Mapping from error code 'index' to atoms.
 */
Eterm error_atom[NUMBER_EXIT_CODES] = {
  am_internal_error,	/* 0 */
  am_normal,		/* 1 */
  am_internal_error,	/* 2 */
  am_badarg,		/* 3 */
  am_badarith,		/* 4 */
  am_badmatch,		/* 5 */
  am_function_clause,	/* 6 */
  am_case_clause,	/* 7 */
  am_if_clause,		/* 8 */
  am_undef,		/* 9 */
  am_badfun,		/* 10 */
  am_badarity,		/* 11 */
  am_timeout_value,	/* 12 */
  am_noproc,		/* 13 */
  am_notalive,		/* 14 */
  am_system_limit,	/* 15 */
  am_try_clause,	/* 16 */
  am_notsup,		/* 17 */
  am_badmap,		/* 18 */
  am_badkey,		/* 19 */
};

/*
 * To fully understand the error handling, one must keep in mind that
 * when an exception is thrown, the search for a handler can jump back
 * and forth between Beam and native code. Upon each mode switch, a
 * dummy handler is inserted so that if an exception reaches that point,
 * the handler is invoked (like any handler) and transfers control so
 * that the search for a real handler is continued in the other mode.
 * Therefore, c_p->freason and c_p->fvalue must still hold the exception
 * info when the handler is executed, but normalized so that creation of
 * error terms and saving of the stack trace is only done once, even if
 * we pass through the error handling code several times.
 *
 * When a new exception is raised, the current stack trace information
 * is quick-saved in a small structure allocated on the heap. Depending
 * on how the exception is eventually caught (perhaps by causing the
 * current process to terminate), the saved information may be used to
 * create a symbolic (human-readable) representation of the stack trace
 * at the point of the original exception.
 */

static BeamInstr*
handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
{
    Eterm* hp;
    Eterm Value = c_p->fvalue;
    Eterm Args = am_true;
    c_p->i = pc;    /* In case we call erts_exit(). */

    ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */

    /*
     * Check if we have an arglist for the top level call. If so, this
     * is encoded in Value, so we have to dig out the real Value as well
     * as the Arglist.
     */
    if (c_p->freason & EXF_ARGLIST) {
	  Eterm* tp;
	  ASSERT(is_tuple(Value));
	  tp = tuple_val(Value);
	  Value = tp[1];
	  Args = tp[2];
    }

    /*
     * Save the stack trace info if the EXF_SAVETRACE flag is set. The
     * main reason for doing this separately is to allow throws to later
     * become promoted to errors without losing the original stack
     * trace, even if they have passed through one or more catch and
     * rethrow. It also makes the creation of symbolic stack traces much
     * more modular.
     */
    if (c_p->freason & EXF_SAVETRACE) {
        save_stacktrace(c_p, pc, reg, bf, Args);
    }

    /*
     * Throws that are not caught are turned into 'nocatch' errors
     */
    if ((c_p->freason & EXF_THROWN) && (c_p->catches <= 0) ) {
	hp = HAlloc(c_p, 3);
        Value = TUPLE2(hp, am_nocatch, Value);
        c_p->freason = EXC_ERROR;
    }

    /* Get the fully expanded error term */
    Value = expand_error_value(c_p, c_p->freason, Value);

    /* Save final error term and stabilize the exception flags so no
       further expansion is done. */
    c_p->fvalue = Value;
    c_p->freason = PRIMARY_EXCEPTION(c_p->freason);

    /* Find a handler or die */
    if ((c_p->catches > 0 || IS_TRACED_FL(c_p, F_EXCEPTION_TRACE))
	&& !(c_p->freason & EXF_PANIC)) {
	BeamInstr *new_pc;
        /* The Beam handler code (catch_end or try_end) checks reg[0]
	   for THE_NON_VALUE to see if the previous code finished
	   abnormally. If so, reg[1], reg[2] and reg[3] should hold the
	   exception class, term and trace, respectively. (If the
	   handler is just a trap to native code, these registers will
	   be ignored.) */
	reg[0] = THE_NON_VALUE;
	reg[1] = exception_tag[GET_EXC_CLASS(c_p->freason)];
	reg[2] = Value;
	reg[3] = c_p->ftrace;
        if ((new_pc = next_catch(c_p, reg))) {
	    c_p->cp = 0;	/* To avoid keeping stale references. */
	    return new_pc;
	}
	if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found");
    }
    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
    terminate_proc(c_p, Value);
    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
    return NULL;
}

/*
 * Find the nearest catch handler
 */
static BeamInstr*
next_catch(Process* c_p, Eterm *reg) {
    int active_catches = c_p->catches > 0;
    int have_return_to_trace = 0;
    Eterm *ptr, *prev, *return_to_trace_ptr = NULL;

    BeamInstr i_return_trace      = beam_return_trace[0];
    BeamInstr i_return_to_trace   = beam_return_to_trace[0];
    BeamInstr i_return_time_trace = beam_return_time_trace[0];

    ptr = prev = c_p->stop;
    ASSERT(is_CP(*ptr));
    ASSERT(ptr <= STACK_START(c_p));
    if (ptr == STACK_START(c_p)) return NULL;
    if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace &&
			     *cp_val(*ptr) != i_return_to_trace &&
			     *cp_val(*ptr) != i_return_time_trace ))
	&& c_p->cp) {
	/* Can not follow cp here - code may be unloaded */
	BeamInstr *cpp = c_p->cp;
	if (cpp == beam_exception_trace) {
	    erts_trace_exception(c_p, cp_val(ptr[0]),
				 reg[1], reg[2], ptr+1);
	    /* Skip return_trace parameters */
	    ptr += 2;
	} else if (cpp == beam_return_trace) {
	    /* Skip return_trace parameters */
	    ptr += 2;
	} else if (cpp == beam_return_time_trace) {
	    /* Skip return_trace parameters */
	    ptr += 1;
	} else if (cpp == beam_return_to_trace) {
	    have_return_to_trace = !0; /* Record next cp */
	}
    }
    while (ptr < STACK_START(c_p)) {
	if (is_catch(*ptr)) {
	    if (active_catches) goto found_catch;
	    ptr++;
	}
	else if (is_CP(*ptr)) {
	    prev = ptr;
	    if (*cp_val(*prev) == i_return_trace) {
		/* Skip stack frame variables */
		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
		    if (is_catch(*ptr) && active_catches) goto found_catch;
		}
		if (cp_val(*prev) == beam_exception_trace) {
		    erts_trace_exception(c_p, cp_val(ptr[0]),
					 reg[1], reg[2], ptr+1);
		}
		/* Skip return_trace parameters */
		ptr += 2;
	    } else if (*cp_val(*prev) == i_return_to_trace) {
		/* Skip stack frame variables */
		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
		    if (is_catch(*ptr) && active_catches) goto found_catch;
		}
		have_return_to_trace = !0; /* Record next cp */
		return_to_trace_ptr = NULL;
	    } else if (*cp_val(*prev) == i_return_time_trace) {
		/* Skip stack frame variables */
		while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) {
		    if (is_catch(*ptr) && active_catches) goto found_catch;
		}
		/* Skip return_trace parameters */
		ptr += 1;
	    } else {
		if (have_return_to_trace) {
		    /* Record this cp as possible return_to trace cp */
		    have_return_to_trace = 0;
		    return_to_trace_ptr = ptr;
		} else return_to_trace_ptr = NULL;
		ptr++;
	    }
	} else ptr++;
    }
    return NULL;
    
 found_catch:
    ASSERT(ptr < STACK_START(c_p));
    c_p->stop = prev;
    if (IS_TRACED_FL(c_p, F_TRACE_RETURN_TO) && return_to_trace_ptr) {
	/* The stackframe closest to the catch contained an
	 * return_to_trace entry, so since the execution now
	 * continues after the catch, a return_to trace message 
	 * would be appropriate.
	 */
	erts_trace_return_to(c_p, cp_val(*return_to_trace_ptr));
    }
    return catch_pc(*ptr);
}

/*
 * Terminating the process when an exception is not caught
 */
static void
terminate_proc(Process* c_p, Eterm Value)
{
    Eterm *hp;
    Eterm Args = NIL;

    /* Add a stacktrace if this is an error. */
    if (GET_EXC_CLASS(c_p->freason) == EXTAG_ERROR) {
        Value = add_stacktrace(c_p, Value, c_p->ftrace);
    }
    /* EXF_LOG is a primary exception flag */
    if (c_p->freason & EXF_LOG) {
	int alive = erts_is_alive;
	erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();

        /* Build the format message */
	erts_dsprintf(dsbufp, "Error in process ~p ");
	if (alive)
	    erts_dsprintf(dsbufp, "on node ~p ");
	erts_dsprintf(dsbufp, "with exit value:~n~p~n");

        /* Build the args in reverse order */
	hp = HAlloc(c_p, 2);
	Args = CONS(hp, Value, Args);
	if (alive) {
	    hp = HAlloc(c_p, 2);
	    Args = CONS(hp, erts_this_node->sysname, Args);
	}
	hp = HAlloc(c_p, 2);
	Args = CONS(hp, c_p->common.id, Args);

	erts_send_error_term_to_logger(c_p->group_leader, dsbufp, Args);
    }
    /*
     * If we use a shared heap, the process will be garbage-collected.
     * Must zero c_p->arity to indicate that there are no live registers.
     */
    c_p->arity = 0;
    erts_do_exit_process(c_p, Value);
}

/*
 * Build and add a symbolic stack trace to the error value.
 */
static Eterm
add_stacktrace(Process* c_p, Eterm Value, Eterm exc) {
    Eterm Where = build_stacktrace(c_p, exc);
    Eterm* hp = HAlloc(c_p, 3);
    return TUPLE2(hp, Value, Where);
}

/*
 * Forming the correct error value from the internal error code.
 * This does not update c_p->fvalue or c_p->freason.
 */
Eterm
expand_error_value(Process* c_p, Uint freason, Eterm Value) {
    Eterm* hp;
    Uint r;

    r = GET_EXC_INDEX(freason);
    ASSERT(r < NUMBER_EXIT_CODES); /* range check */
    ASSERT(is_value(Value));

    switch (r) {
    case (GET_EXC_INDEX(EXC_PRIMARY)):
        /* Primary exceptions use fvalue as it is */
	break;
    case (GET_EXC_INDEX(EXC_BADMATCH)):
    case (GET_EXC_INDEX(EXC_CASE_CLAUSE)):
    case (GET_EXC_INDEX(EXC_TRY_CLAUSE)):
    case (GET_EXC_INDEX(EXC_BADFUN)):
    case (GET_EXC_INDEX(EXC_BADARITY)):
    case (GET_EXC_INDEX(EXC_BADMAP)):
    case (GET_EXC_INDEX(EXC_BADKEY)):
        /* Some common exceptions: value -> {atom, value} */
        ASSERT(is_value(Value));
	hp = HAlloc(c_p, 3);
	Value = TUPLE2(hp, error_atom[r], Value);
	break;
    default:
        /* Other exceptions just use an atom as descriptor */
        Value = error_atom[r];
	break;
    }
#ifdef DEBUG
    ASSERT(Value != am_internal_error);
#endif
# 5682 "beam/beam_emu.c"
    return Value;
}

/*
 * Quick-saving the stack trace in an internal form on the heap. Note
 * that c_p->ftrace will point to a cons cell which holds the given args
 * and the saved data (encoded as a bignum).
 *
 * There is an issue with line number information. Line number
 * information is associated with the address *before* an operation
 * that may fail or be stored stored on the stack. But continuation
 * pointers point after its call instruction, not before. To avoid
 * finding the wrong line number, we'll need to adjust them so that
 * they point at the beginning of the call instruction or inside the
 * call instruction. Since its impractical to point at the beginning,
 * we'll do the simplest thing and decrement the continuation pointers
 * by one.
 *
 * Here is an example of what can go wrong. Without the adjustment
 * of continuation pointers, the call at line 42 below would seem to
 * be at line 43:
 *
 * line 42
 * call ...
 * line 43
 * gc_bif ...
 *
 * (It would be much better to put the arglist - when it exists - in the
 * error value instead of in the actual trace; e.g. '{badarg, Args}'
 * instead of using 'badarg' with Args in the trace. The arglist may
 * contain very large values, and right now they will be kept alive as
 * long as the stack trace is live. Preferably, the stack trace should
 * always be small, so that it does not matter if it is long-lived.
 * However, it is probably not possible to ever change the format of
 * error terms.)
 */

static void
save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
		Eterm args) {
    struct StackTrace* s;
    int sz;
    int depth = erts_backtrace_depth;    /* max depth (never negative) */
    if (depth > 0) {
	/* There will always be a current function */
	depth --;
    }

    /* Create a container for the exception data */
    sz = (offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth
          + sizeof(Eterm) - 1) / sizeof(Eterm);
    s = (struct StackTrace *) HAlloc(c_p, 1 + sz);
    /* The following fields are inside the bignum */
    s->header = make_pos_bignum_header(sz);
    s->freason = c_p->freason;
    s->depth = 0;

    /*
     * If the failure was in a BIF other than 'error', 'exit' or
     * 'throw', find the bif-table index and save the argument
     * registers by consing up an arglist.
     */
    if (bf != NULL && bf != error_1 && bf != error_2 &&
	bf != exit_1 && bf != throw_1) {
        int i;
	int a = 0;
	for (i = 0; i < BIF_SIZE; i++) {
	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
		Export *ep = bif_export[i];
		s->current = ep->code;
	        a = bif_table[i].arity;
		break;
	    }
	}
	if (i >= BIF_SIZE) {
	    /* 
	     * The Bif does not really exist (no BIF entry).  It is a
	     * TRAP and traps are called through apply_bif, which also
	     * sets c_p->current (luckily).
	     * OR it is a NIF called by call_nif where current is also set.
	     */
	    ASSERT(c_p->current);
	    s->current = c_p->current;
	    a = s->current[2];
	}
	/* Save first stack entry */
	ASSERT(pc);
	if (depth > 0) {
	    s->trace[s->depth++] = pc;
	    depth--;
	}
	/* Save second stack entry if CP is valid and different from pc */
	if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) {
	    s->trace[s->depth++] = c_p->cp - 1;
	    depth--;
	}
	s->pc = NULL;
	args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
    } else {
	s->current = c_p->current;
        /* 
	 * For a function_clause error, the arguments are in the beam
	 * registers, c_p->cp is valid, and c_p->current is set.
	 */
	if ( (GET_EXC_INDEX(s->freason)) ==
	     (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) {
	    int a;
	    ASSERT(s->current);
	    a = s->current[2];
	    args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
	    /* Save first stack entry */
	    ASSERT(c_p->cp);
	    if (depth > 0) {
		s->trace[s->depth++] = c_p->cp - 1;
		depth--;
	    }
	    s->pc = NULL; /* Ignore pc */
	} else {
	    if (depth > 0 && c_p->cp != 0 && c_p->cp != pc) {
		s->trace[s->depth++] = c_p->cp - 1;
		depth--;
	    }
	    s->pc = pc;
	}
    }

    /* Package args and stack trace */
    {
	Eterm *hp;
	hp = HAlloc(c_p, 2);
	c_p->ftrace = CONS(hp, args, make_big((Eterm *) s));
    }

    /* Save the actual stack trace */
    erts_save_stacktrace(c_p, s, depth);
}

void
erts_save_stacktrace(Process* p, struct StackTrace* s, int depth)
{
    if (depth > 0) {
	Eterm *ptr;
	BeamInstr *prev = s->depth ? s->trace[s->depth-1] : NULL;
	BeamInstr i_return_trace = beam_return_trace[0];
	BeamInstr i_return_to_trace = beam_return_to_trace[0];

	/*
	 * Traverse the stack backwards and add all unique continuation
	 * pointers to the buffer, up to the maximum stack trace size.
	 * 
	 * Skip trace stack frames.
	 */
	ptr = p->stop;
	if (ptr < STACK_START(p) &&
	    (is_not_CP(*ptr)|| (*cp_val(*ptr) != i_return_trace &&
				*cp_val(*ptr) != i_return_to_trace)) &&
	    p->cp) {
	    /* Cannot follow cp here - code may be unloaded */
	    BeamInstr *cpp = p->cp;
	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
		/* Skip return_trace parameters */
		ptr += 2;
	    } else if (cpp == beam_return_to_trace) {
		/* Skip return_to_trace parameters */
		ptr += 1;
	    }
	}
	while (ptr < STACK_START(p) && depth > 0) {
	    if (is_CP(*ptr)) {
		if (*cp_val(*ptr) == i_return_trace) {
		    /* Skip stack frame variables */
		    do ++ptr; while (is_not_CP(*ptr));
		    /* Skip return_trace parameters */
		    ptr += 2;
		} else if (*cp_val(*ptr) == i_return_to_trace) {
		    /* Skip stack frame variables */
		    do ++ptr; while (is_not_CP(*ptr));
		} else {
		    BeamInstr *cp = cp_val(*ptr);
		    if (cp != prev) {
			/* Record non-duplicates only */
			prev = cp;
			s->trace[s->depth++] = cp - 1;
			depth--;
		    }
		    ptr++;
		}
	    } else ptr++;
	}
    }
}

/*
 * Getting the relevant fields from the term pointed to by ftrace
 */

static struct StackTrace *get_trace_from_exc(Eterm exc) {
    if (exc == NIL) {
	return NULL;
    } else {
	ASSERT(is_list(exc));
	return (struct StackTrace *) big_val(CDR(list_val(exc)));
    }
}

static Eterm get_args_from_exc(Eterm exc) {
    if (exc == NIL) {
	return NIL;
    } else {
	ASSERT(is_list(exc));
	return CAR(list_val(exc));
    }
}

static int is_raised_exc(Eterm exc) {
    if (exc == NIL) {
        return 0;
    } else {
        ASSERT(is_list(exc));
        return bignum_header_is_neg(*big_val(CDR(list_val(exc))));
    }
}

/*
 * Creating a list with the argument registers
 */
static Eterm
make_arglist(Process* c_p, Eterm* reg, int a) {
    Eterm args = NIL;
    Eterm* hp = HAlloc(c_p, 2*a);
    while (a > 0) {
        args = CONS(hp, reg[a-1], args);
	hp += 2;
	a--;
    }
    return args;
}

/*
 * Building a symbolic representation of a saved stack trace. Note that
 * the exception object 'exc', unless NIL, points to a cons cell which
 * holds the given args and the quick-saved data (encoded as a bignum).
 *
 * If the bignum is negative, the given args is a complete stacktrace.
 */
Eterm
build_stacktrace(Process* c_p, Eterm exc) {
    struct StackTrace* s;
    Eterm  args;
    int    depth;
    FunctionInfo fi;
    FunctionInfo* stk;
    FunctionInfo* stkp;
    Eterm res = NIL;
    Uint heap_size;
    Eterm* hp;
    Eterm mfa;
    int i;

    if (! (s = get_trace_from_exc(exc))) {
        return NIL;
    }
#ifdef HIPE
    if (s->freason & EXF_NATIVE) {
	return hipe_build_stacktrace(c_p, s);
    }
#endif
# 5949 "beam/beam_emu.c"
    if (is_raised_exc(exc)) {
	return get_args_from_exc(exc);
    }

    /*
     * Find the current function. If the saved s->pc is null, then the
     * saved s->current should already contain the proper value.
     */
    if (s->pc != NULL) {
	erts_lookup_function_info(&fi, s->pc, 1);
    } else if (GET_EXC_INDEX(s->freason) ==
	       GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) {
	erts_lookup_function_info(&fi, s->current, 1);
    } else {
	erts_set_current_function(&fi, s->current);
    }

    /*
     * If fi.current is still NULL, default to the initial function
     * (e.g. spawn_link(erlang, abs, [1])).
     */
    if (fi.current == NULL) {
	erts_set_current_function(&fi, c_p->u.initial);
	args = am_true; /* Just in case */
    } else {
	args = get_args_from_exc(exc);
    }

    /*
     * Look up all saved continuation pointers and calculate
     * needed heap space.
     */
    depth = s->depth;
    stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP,
				      depth*sizeof(FunctionInfo));
    heap_size = fi.needed + 2;
    for (i = 0; i < depth; i++) {
	erts_lookup_function_info(stkp, s->trace[i], 1);
	if (stkp->current) {
	    heap_size += stkp->needed + 2;
	    stkp++;
	}
    }

    /*
     * Allocate heap space and build the stacktrace.
     */
    hp = HAlloc(c_p, heap_size);
    while (stkp > stk) {
	stkp--;
	hp = erts_build_mfa_item(stkp, hp, am_true, &mfa);
	res = CONS(hp, mfa, res);
	hp += 2;
    }
    hp = erts_build_mfa_item(&fi, hp, args, &mfa);
    res = CONS(hp, mfa, res);

    erts_free(ERTS_ALC_T_TMP, (void *) stk);
    return res;
}

static BeamInstr*
call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
{
    Eterm* hp;
    Export* ep;
    int arity;
    Eterm args;
    Uint sz;
    int i;

    /*
     * Search for the error_handler module.
     */
    ep = erts_find_function(erts_proc_get_error_handler(p), func, 3,
			    erts_active_code_ix());
    if (ep == NULL) {		/* No error handler */
	p->current = fi;
	p->freason = EXC_UNDEF;
	return 0;
    }

    /*
     * Create a list with all arguments in the x registers.
     */

    arity = fi[2];
    sz = 2 * arity;
    if (HeapWordsLeft(p) < sz) {
	erts_garbage_collect(p, sz, reg, arity);
    }
    hp = HEAP_TOP(p);
    HEAP_TOP(p) += sz;
    args = NIL;
    for (i = arity-1; i >= 0; i--) {
	args = CONS(hp, reg[i], args);
	hp += 2;
    }

    /*
     * Set up registers for call to error_handler:<func>/3.
     */
    reg[0] = fi[0];
    reg[1] = fi[1];
    reg[2] = args;
    return ep->addressv[erts_active_code_ix()];
}

static Export*
apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg)
{
    Export* ep;

    /*
     * Find the export table index for the error handler. Return NULL if
     * there is no error handler module.
     */

    if ((ep = erts_active_export_entry(erts_proc_get_error_handler(p),
				     am_undefined_function, 3)) == NULL) {
	return NULL;
    } else {
	int i;
	Uint sz = 2*arity;
	Eterm* hp;
	Eterm args = NIL;
	
	/*
	 * Always copy args from registers to a new list; this ensures
	 * that we have the same behaviour whether or not this was
	 * called from apply or fixed_apply (any additional last
	 * THIS-argument will be included, assuming that arity has been
	 * properly adjusted).
	 */

	if (HeapWordsLeft(p) < sz) {
	    erts_garbage_collect(p, sz, reg, arity);
	}
	hp = HEAP_TOP(p);
	HEAP_TOP(p) += sz;
	for (i = arity-1; i >= 0; i--) {
	    args = CONS(hp, reg[i], args);
	    hp += 2;
	}
	reg[0] = module;
	reg[1] = function;
	reg[2] = args;
    }

    return ep;
}

static BeamInstr*
apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
{
    int arity;
    Export* ep;
    Eterm tmp, this;

    /*
     * Check the arguments which should be of the form apply(Module,
     * Function, Arguments) where Function is an atom and
     * Arguments is an arity long list of terms.
     */
    if (is_not_atom(function)) {
	/*
	 * No need to test args here -- done below.
	 */
    error:
	p->freason = BADARG;

    error2:
	reg[0] = module;
	reg[1] = function;
	reg[2] = args;
	return 0;
    }

    /* The module argument may be either an atom or an abstract module
     * (currently implemented using tuples, but this might change).
     */
    this = THE_NON_VALUE;
    if (is_not_atom(module)) {
	Eterm* tp;

        if (is_not_tuple(module)) goto error;
        tp = tuple_val(module);
        if (arityval(tp[0]) < 1) goto error;
        this = module;
        module = tp[1];
        if (is_not_atom(module)) goto error;
    }
    
    /*
     * Walk down the 3rd parameter of apply (the argument list) and copy
     * the parameters to the x registers (reg[]). If the module argument
     * was an abstract module, add 1 to the function arity and put the
     * module argument in the n+1st x register as a THIS reference.
     */

    tmp = args;
    arity = 0;
    while (is_list(tmp)) {
	if (arity < (MAX_REG - 1)) {
	    reg[arity++] = CAR(list_val(tmp));
	    tmp = CDR(list_val(tmp));
	} else {
	    p->freason = SYSTEM_LIMIT;
	    goto error2;
	}
    }
    if (is_not_nil(tmp)) {	/* Must be well-formed list */
	goto error;
    }
    if (this != THE_NON_VALUE) {
        reg[arity++] = this;
    }

    /*
     * Get the index into the export table, or failing that the export
     * entry for the error handler.
     *
     * Note: All BIFs have export entries; thus, no special case is needed.
     */

    if ((ep = erts_active_export_entry(module, function, arity)) == NULL) {
	if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL) goto error;
    } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
	save_calls(p, ep);
    }
    DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
    return ep->addressv[erts_active_code_ix()];
}

static BeamInstr*
fixed_apply(Process* p, Eterm* reg, Uint arity)
{
    Export* ep;
    Eterm module;
    Eterm function;

    module = reg[arity];    /* The THIS pointer already in place */
    function = reg[arity+1];

    if (is_not_atom(function)) {
    error:
	p->freason = BADARG;
	reg[0] = module;
	reg[1] = function;
	reg[2] = NIL;
	return 0;
    }

    /* The module argument may be either an atom or an abstract module
     * (currently implemented using tuples, but this might change).
     */
    if (is_not_atom(module)) {
	Eterm* tp;
        if (is_not_tuple(module)) goto error;
        tp = tuple_val(module);
        if (arityval(tp[0]) < 1) goto error;
        module = tp[1];
        if (is_not_atom(module)) goto error;
        ++arity;
    }
    
    /*
     * Get the index into the export table, or failing that the export
     * entry for the error handler module.
     *
     * Note: All BIFs have export entries; thus, no special case is needed.
     */

    if ((ep = erts_active_export_entry(module, function, arity)) == NULL) {
	if ((ep = apply_setup_error_handler(p, module, function, arity, reg)) == NULL)
	    goto error;
    } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
	save_calls(p, ep);
    }
    DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
    return ep->addressv[erts_active_code_ix()];
}

int
erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg)
{
    int arity;
    Eterm tmp;

#ifndef ERTS_SMP
    if (ERTS_PROC_IS_EXITING(c_p)) {
	/*
	 * I non smp case:
	 *
	 * Currently executing process might be sent an exit
	 * signal if it is traced by a port that it also is
	 * linked to, and the port terminates during the
	 * trace. In this case we do *not* want to clear
	 * the active flag, which will make the process hang
	 * in limbo forever. Get out of here and terminate
	 * the process...
	 */
	return -1;
    }
#endif
# 6254 "beam/beam_emu.c"

    if (is_not_atom(module) || is_not_atom(function)) {
	/*
	 * No need to test args here -- done below.
	 */
    error:
	c_p->freason = BADARG;

    error2:
	reg[0] = module;
	reg[1] = function;
	reg[2] = args;
	return 0;
    }

    arity = 0;
    tmp = args;
    while (is_list(tmp)) {
	if (arity < MAX_REG) {
	    tmp = CDR(list_val(tmp));
	    arity++;
	} else {
	    c_p->freason = SYSTEM_LIMIT;
	    goto error2;
	}
    }
    if (is_not_nil(tmp)) {	/* Must be well-formed list */
	goto error;
    }

    /*
     * At this point, arguments are known to be good.
     */

    if (c_p->arg_reg != c_p->def_arg_reg) {
	/* Save some memory */
	erts_free(ERTS_ALC_T_ARG_REG, c_p->arg_reg);
	c_p->arg_reg = c_p->def_arg_reg;
	c_p->max_arg_reg = sizeof(c_p->def_arg_reg)/sizeof(c_p->def_arg_reg[0]);
    }

#ifdef USE_VM_PROBES
    if (DTRACE_ENABLED(process_hibernate)) {
        DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE);
        DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE);
        dtrace_fun_decode(c_p, module, function, arity,
                          process_name, mfa);
        DTRACE2(process_hibernate, process_name, mfa);
    }
#endif
# 6304 "beam/beam_emu.c"
    /*
     * Arrange for the process to be resumed at the given MFA with
     * the stack cleared.
     */
    c_p->arity = 3;
    c_p->arg_reg[0] = module;
    c_p->arg_reg[1] = function;
    c_p->arg_reg[2] = args;
    c_p->stop = STACK_START(c_p);
    c_p->catches = 0;
    c_p->i = beam_apply;
    c_p->cp = (BeamInstr *) beam_apply+1;

    /*
     * If there are no waiting messages, garbage collect and
     * shrink the heap. 
     */
    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
    ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
    if (!c_p->msg.len) {
	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
	c_p->fvalue = NIL;
	PROCESS_MAIN_CHK_LOCKS(c_p);
	erts_garbage_collect_hibernate(c_p);
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
	PROCESS_MAIN_CHK_LOCKS(c_p);
	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
#ifndef ERTS_SMP
	if (ERTS_PROC_IS_EXITING(c_p)) {
	    /*
	     * See comment in the begining of the function...
	     *
	     * This second test is needed since gc might be traced.
	     */
	    return -1;
	}
#else /* ERTS_SMP */
# 6341 "beam/beam_emu.c"
	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
	if (!c_p->msg.len)
#endif
# 6344 "beam/beam_emu.c"
	    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_ACTIVE);
	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
    }
    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
    c_p->current = bif_export[BIF_hibernate_3]->code;
    c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */
    return 1;
}

static BeamInstr*
call_fun(Process* p,		/* Current process. */
	 int arity,		/* Number of arguments for Fun. */
	 Eterm* reg,		/* Contents of registers. */
	 Eterm args)		/* THE_NON_VALUE or pre-built list of arguments. */
{
    Eterm fun = reg[arity];
    Eterm hdr;
    int i;
    Eterm* hp;

    if (!is_boxed(fun)) {
	goto badfun;
    }
    hdr = *boxed_val(fun);

    if (is_fun_header(hdr)) {
	ErlFunThing* funp = (ErlFunThing *) fun_val(fun);
	ErlFunEntry* fe;
	BeamInstr* code_ptr;
	Eterm* var_ptr;
	int actual_arity;
	unsigned num_free;

	fe = funp->fe;
	num_free = funp->num_free;
	code_ptr = fe->address;
	actual_arity = (int) code_ptr[-1];

	if (actual_arity == arity+num_free) {
	    DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3],
			(Eterm)code_ptr[-2],
			code_ptr[-1]);
	    if (num_free == 0) {
		return code_ptr;
	    } else {
		var_ptr = funp->env;
		reg += arity;
		i = 0;
		do {
		    reg[i] = var_ptr[i];
		    i++;
		} while (i < num_free);
		reg[i] = fun;
		return code_ptr;
	    }
	    return code_ptr;
	} else {
	    /*
	     * Something wrong here. First build a list of the arguments.
	     */

	    if (is_non_value(args)) {
		Uint sz = 2 * arity;
		args = NIL;
		if (HeapWordsLeft(p) < sz) {
		    erts_garbage_collect(p, sz, reg, arity+1);
		    fun = reg[arity];
		}
		hp = HEAP_TOP(p);
		HEAP_TOP(p) += sz;
		for (i = arity-1; i >= 0; i--) {
		    args = CONS(hp, reg[i], args);
		    hp += 2;
		}
	    }

	    if (actual_arity >= 0) {
		/*
		 * There is a fun defined, but the call has the wrong arity.
		 */
		hp = HAlloc(p, 3);
		p->freason = EXC_BADARITY;
		p->fvalue = TUPLE2(hp, fun, args);
		return NULL;
	    } else {
		Export* ep;
		Module* modp;
		Eterm module;
		ErtsCodeIndex code_ix = erts_active_code_ix();

		/*
		 * No arity. There is no module loaded that defines the fun,
		 * either because the fun is newly created from the external
		 * representation (the module has never been loaded),
		 * or the module defining the fun has been unloaded.
		 */
		module = fe->module;
		if ((modp = erts_get_module(module, code_ix)) != NULL
		    && modp->curr.code != NULL) {
		    /*
		     * There is a module loaded, but obviously the fun is not
		     * defined in it. We must not call the error_handler
		     * (or we will get into an infinite loop).
		     */
		    goto badfun;
		}
		
		/*
		 * No current code for this module. Call the error_handler module
		 * to attempt loading the module.
		 */

		ep = erts_find_function(erts_proc_get_error_handler(p),
					am_undefined_lambda, 3, code_ix);
		if (ep == NULL) {	/* No error handler */
		    p->current = NULL;
		    p->freason = EXC_UNDEF;
		    return NULL;
		}
		reg[0] = module;
		reg[1] = fun;
		reg[2] = args;
		reg[3] = NIL;
		return ep->addressv[erts_active_code_ix()];
	    }
	}
    } else if (is_export_header(hdr)) {
	Export *ep;
	int actual_arity;

	ep = *((Export **) (export_val(fun) + 1));
	actual_arity = (int) ep->code[2];

	if (arity == actual_arity) {
	    DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]);
	    return ep->addressv[erts_active_code_ix()];
	} else {
	    /*
	     * Wrong arity. First build a list of the arguments.
	     */  

	    if (is_non_value(args)) {
		args = NIL;
		hp = HAlloc(p, arity*2);
		for (i = arity-1; i >= 0; i--) {
		    args = CONS(hp, reg[i], args);
		    hp += 2;
		}
	    }

	    hp = HAlloc(p, 3);
	    p->freason = EXC_BADARITY;
	    p->fvalue = TUPLE2(hp, fun, args);
	    return NULL;
	}
    } else {
    badfun:
	p->current = NULL;
	p->freason = EXC_BADFUN;
	p->fvalue = fun;
	return NULL;
    }
}

static BeamInstr*
apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg)
{
    int arity;
    Eterm tmp;

    /*
     * Walk down the 3rd parameter of apply (the argument list) and copy
     * the parameters to the x registers (reg[]).
     */

    tmp = args;
    arity = 0;
    while (is_list(tmp)) {
	if (arity < MAX_REG-1) {
	    reg[arity++] = CAR(list_val(tmp));
	    tmp = CDR(list_val(tmp));
	} else {
	    p->freason = SYSTEM_LIMIT;
	    return NULL;
	}
    }

    if (is_not_nil(tmp)) {	/* Must be well-formed list */
	p->freason = EXC_UNDEF;
	return NULL;
    }
    reg[arity] = fun;
    return call_fun(p, arity, reg, args);
}



static Eterm
new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
{
    unsigned needed = ERL_FUN_SIZE + num_free;
    ErlFunThing* funp;
    Eterm* hp;
    int i;

    if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) {
	PROCESS_MAIN_CHK_LOCKS(p);
	erts_garbage_collect(p, needed, reg, num_free);
	ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
	PROCESS_MAIN_CHK_LOCKS(p);
    }
    hp = p->htop;
    p->htop = hp + needed;
    funp = (ErlFunThing *) hp;
    hp = funp->env;
    erts_refc_inc(&fe->refc, 2);
    funp->thing_word = HEADER_FUN;
    funp->next = MSO(p).first;
    MSO(p).first = (struct erl_off_heap_header*) funp;
    funp->fe = fe;
    funp->num_free = num_free;
    funp->creator = p->common.id;
#ifdef HIPE
    funp->native_address = fe->native_address;
#endif
# 6569 "beam/beam_emu.c"
    funp->arity = (int)fe->address[-1] - num_free;
    for (i = 0; i < num_free; i++) {
	*hp++ = reg[i];
    }
    return make_fun(funp);
}

static Eterm get_map_element(Eterm map, Eterm key)
{
    Uint32 hx;
    const Eterm *vs;
    if (is_flatmap(map)) {
	flatmap_t *mp;
	Eterm *ks;
	Uint i;
	Uint n;

	mp = (flatmap_t *)flatmap_val(map);
	ks = flatmap_get_keys(mp);
	vs = flatmap_get_values(mp);
	n  = flatmap_get_size(mp);
	if (is_immed(key)) {
	    for (i = 0; i < n; i++) {
		if (ks[i] == key) {
		    return vs[i];
		}
	    }
	} else {
	    for (i = 0; i < n; i++) {
		if (EQ(ks[i], key)) {
		    return vs[i];
		}
	    }
	}
	return THE_NON_VALUE;
    }
    ASSERT(is_hashmap(map));
    hx = hashmap_make_hash(key);
    vs = erts_hashmap_get(hx,key,map);
    return vs ? *vs : THE_NON_VALUE;
}

static Eterm get_map_element_hash(Eterm map, Eterm key, Uint32 hx)
{
    const Eterm *vs;

    if (is_flatmap(map)) {
	flatmap_t *mp;
	Eterm *ks;
	Uint i;
	Uint n;

	mp = (flatmap_t *)flatmap_val(map);
	ks = flatmap_get_keys(mp);
	vs = flatmap_get_values(mp);
	n  = flatmap_get_size(mp);
	if (is_immed(key)) {
	    for (i = 0; i < n; i++) {
		if (ks[i] == key) {
		    return vs[i];
		}
	    }
	} else {
	    for (i = 0; i < n; i++) {
		if (EQ(ks[i], key)) {
		    return vs[i];
		}
	    }
	}
	return THE_NON_VALUE;
    }

    ASSERT(is_hashmap(map));
    ASSERT(hx == hashmap_make_hash(key));
    vs = erts_hashmap_get(hx, key, map);
    return vs ? *vs : THE_NON_VALUE;
}

#define GET_TERM(term, dest)					\
do {								\
    Eterm src = (Eterm)(term);					\
    switch (src & _TAG_IMMED1_MASK) {				\
    case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
	dest = x(0);						\
	break;							\
    case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
	dest = x(src >> _TAG_IMMED1_SIZE);			\
	break;							\
    case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER:	\
	dest = y(src >> _TAG_IMMED1_SIZE);			\
	break;							\
    default:							\
	dest = src;						\
	break;							\
    }								\
} while(0)


static Eterm
new_map(Process* p, Eterm* reg, BeamInstr* I)
{
    Uint n = Arg(3);
    Uint i;
    Uint need = n + 1 /* hdr */ + 1 /*size*/ + 1 /* ptr */ + 1 /* arity */;
    Eterm keys;
    Eterm *mhp,*thp;
    Eterm *E;
    BeamInstr *ptr;
    flatmap_t *mp;
    ErtsHeapFactory factory;

    ptr = &Arg(4);

    if (n > 2*MAP_SMALL_MAP_LIMIT) {
        Eterm res;
	if (HeapWordsLeft(p) < n) {
	    erts_garbage_collect(p, n, reg, Arg(2));
	}

	mhp = p->htop;
	thp = p->htop;
	E   = p->stop;

	for (i = 0; i < n/2; i++) {
	    GET_TERM(*ptr++, *mhp++);
	    GET_TERM(*ptr++, *mhp++);
	}

	p->htop = mhp;

        erts_factory_proc_init(&factory, p);
        res = erts_hashmap_from_array(&factory, thp, n/2, 0);
        erts_factory_close(&factory);
        if (p->mbuf) {
            Uint live = Arg(2);
            reg[live] = res;
            erts_garbage_collect(p, 0, reg, live+1);
            res       = reg[live];
            E = p->stop;
        }
        return res;
    }

    if (HeapWordsLeft(p) < need) {
	erts_garbage_collect(p, need, reg, Arg(2));
    }

    thp    = p->htop;
    mhp    = thp + 1 + n/2;
    E      = p->stop;
    keys   = make_tuple(thp);
    *thp++ = make_arityval(n/2);

    mp = (flatmap_t *)mhp; mhp += MAP_HEADER_FLATMAP_SZ;
    mp->thing_word = MAP_HEADER_FLATMAP;
    mp->size = n/2;
    mp->keys = keys;

    for (i = 0; i < n/2; i++) {
	GET_TERM(*ptr++, *thp++);
	GET_TERM(*ptr++, *mhp++);
    }
    p->htop = mhp;
    return make_flatmap(mp);
}

static Eterm
update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
{
    Uint n;
    Uint num_old;
    Uint num_updates;
    Uint need;
    flatmap_t *old_mp, *mp;
    Eterm res;
    Eterm* hp;
    Eterm* E;
    Eterm* old_keys;
    Eterm* old_vals;
    BeamInstr* new_p;
    Eterm new_key;
    Eterm* kp;

    new_p = &Arg(5);
    num_updates = Arg(4) / 2;

    if (is_not_flatmap(map)) {
	Uint32 hx;
	Eterm val;

	/* apparently the compiler does not emit is_map instructions,
	 * bad compiler */

	if (is_not_hashmap(map))
	    return THE_NON_VALUE;

	res = map;
	E = p->stop;
	while(num_updates--) {
	    /* assoc can't fail */
	    GET_TERM(new_p[0], new_key);
	    GET_TERM(new_p[1], val);
	    hx = hashmap_make_hash(new_key);

	    res = erts_hashmap_insert(p, hx, new_key, val, res,  0);
	    if (p->mbuf) {
		Uint live = Arg(3);
		reg[live] = res;
		erts_garbage_collect(p, 0, reg, live+1);
		res       = reg[live];
		E = p->stop;
	    }

	    new_p += 2;
	}
	return res;
    }

    old_mp  = (flatmap_t *) flatmap_val(map);
    num_old = flatmap_get_size(old_mp);

    /*
     * If the old map is empty, create a new map.
     */

    if (num_old == 0) {
	return new_map(p, reg, I+1);
    }

    /*
     * Allocate heap space for the worst case (i.e. all keys in the
     * update list are new).
     */

    need = 2*(num_old+num_updates) + 1 + MAP_HEADER_FLATMAP_SZ;
    if (HeapWordsLeft(p) < need) {
	Uint live = Arg(3);
	reg[live] = map;
	erts_garbage_collect(p, need, reg, live+1);
	map      = reg[live];
	old_mp   = (flatmap_t *)flatmap_val(map);
    }

    /*
     * Build the skeleton for the map, ready to be filled in.
     *
     * +-----------------------------------+
     * | (Space for aritvyal for keys)     | <-----------+
     * +-----------------------------------+		 |
     * | (Space for key 1)		   |		 |    <-- kp
     * +-----------------------------------+		 |
     *        .				    		 |
     *        .				    		 |
     *        .				    		 |
     * +-----------------------------------+		 |
     * | (Space for last key)		   |		 |
     * +-----------------------------------+		 |
     * | MAP_HEADER			   |		 |
     * +-----------------------------------+		 |
     * | (Space for number of keys/values) |		 |
     * +-----------------------------------+		 |
     * | Boxed tuple pointer            >----------------+
     * +-----------------------------------+
     * | (Space for value 1)		   |                  <-- hp
     * +-----------------------------------+
     */

    E = p->stop;
    kp = p->htop + 1;		/* Point to first key */
    hp = kp + num_old + num_updates;

    res = make_flatmap(hp);
    mp = (flatmap_t *)hp;
    hp += MAP_HEADER_FLATMAP_SZ;
    mp->thing_word = MAP_HEADER_FLATMAP;
    mp->keys = make_tuple(kp-1);

    old_vals = flatmap_get_values(old_mp);
    old_keys = flatmap_get_keys(old_mp);

    GET_TERM(*new_p, new_key);
    n = num_updates;

    /*
     * Fill in keys and values, until we run out of either updates
     * or old values and keys.
     */

    for (;;) {
	Eterm key;
	Sint c;

	ASSERT(kp < (Eterm *)mp);
	key = *old_keys;
	if ((c = CMP_TERM(key, new_key)) < 0) {
	    /* Copy old key and value */
	    *kp++ = key;
	    *hp++ = *old_vals;
	    old_keys++, old_vals++, num_old--;
	} else {		/* Replace or insert new */
	    GET_TERM(new_p[1], *hp++);
	    if (c > 0) {	/* If new new key */
		*kp++ = new_key;
	    } else {		/* If replacement */
		*kp++ = key;
		old_keys++, old_vals++, num_old--;
	    }
	    n--;
	    if (n == 0) {
		break;
	    } else {
		new_p += 2;
		GET_TERM(*new_p, new_key);
	    }
	}
	if (num_old == 0) {
	    break;
	}
    }

    /*
     * At this point, we have run out of either old keys and values,
     * or the update list. In other words, at least of one n and
     * num_old must be zero.
     */

    if (n > 0) {
	/*
	 * All old keys and values have been copied, but there
	 * are still new keys and values in the update list that
	 * must be copied.
	 */
	ASSERT(num_old == 0);
	while (n-- > 0) {
	    GET_TERM(new_p[0], *kp++);
	    GET_TERM(new_p[1], *hp++);
	    new_p += 2;
	}
    } else {
	/*
	 * All updates are now done. We may still have old
	 * keys and values that we must copy.
	 */
	ASSERT(n == 0);
	while (num_old-- > 0) {
	    ASSERT(kp < (Eterm *)mp);
	    *kp++ = *old_keys++;
	    *hp++ = *old_vals++;
	}
    }

    /*
     * Calculate how many values that are unused at the end of the
     * key tuple and fill it out with a bignum header.
     */
    if ((n = (Eterm *)mp - kp) > 0) {
	*kp = make_pos_bignum_header(n-1);
    }

    /*
     * Fill in the size of the map in both the key tuple and in the map.
     */

    n = kp - p->htop - 1;	/* Actual number of keys/values */
    *p->htop = make_arityval(n);
    p->htop  = hp;
    mp->size = n;

    /* The expensive case, need to build a hashmap */
    if (n > MAP_SMALL_MAP_LIMIT) {
	res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n);
	if (p->mbuf) {
	    Uint live = Arg(3);
	    reg[live] = res;
	    erts_garbage_collect(p, 0, reg, live+1);
	    res       = reg[live];
	}
    }
    return res;
}

/*
 * Update values for keys that already exist in the map.
 */

static Eterm
update_map_exact(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
{
    Uint n;
    Uint i;
    Uint num_old;
    Uint need;
    flatmap_t *old_mp, *mp;
    Eterm res;
    Eterm* hp;
    Eterm* E;
    Eterm* old_keys;
    Eterm* old_vals;
    BeamInstr* new_p;
    Eterm new_key;

    new_p = &Arg(5);
    n = Arg(4) / 2;		/* Number of values to be updated */
    ASSERT(n > 0);

    if (is_not_flatmap(map)) {
	Uint32 hx;
	Eterm val;

	/* apparently the compiler does not emit is_map instructions,
	 * bad compiler */

	if (is_not_hashmap(map)) {
	    p->freason = BADMAP;
	    p->fvalue = map;
	    return THE_NON_VALUE;
	}

	res = map;
	E = p->stop;
	while(n--) {
	    GET_TERM(new_p[0], new_key);
	    GET_TERM(new_p[1], val);
	    hx = hashmap_make_hash(new_key);

	    res = erts_hashmap_insert(p, hx, new_key, val, res,  1);
	    if (is_non_value(res)) {
		p->fvalue = new_key;
		p->freason = BADKEY;
		return res;
	    }

	    if (p->mbuf) {
		Uint live = Arg(3);
		reg[live] = res;
		erts_garbage_collect(p, 0, reg, live+1);
		res       = reg[live];
		E = p->stop;
	    }

	    new_p += 2;
	}
	return res;
    }

    old_mp = (flatmap_t *) flatmap_val(map);
    num_old = flatmap_get_size(old_mp);

    /*
     * If the old map is empty, fail.
     */

    if (num_old == 0) {
	E = p->stop;
	p->freason = BADKEY;
	GET_TERM(new_p[0], p->fvalue);
	return THE_NON_VALUE;
    }

    /*
     * Allocate the exact heap space needed.
     */

    need = num_old + MAP_HEADER_FLATMAP_SZ;
    if (HeapWordsLeft(p) < need) {
	Uint live = Arg(3);
	reg[live] = map;
	erts_garbage_collect(p, need, reg, live+1);
	map      = reg[live];
	old_mp   = (flatmap_t *)flatmap_val(map);
    }

    /*
     * Update map, keeping the old key tuple.
     */

    hp = p->htop;
    E = p->stop;

    old_vals = flatmap_get_values(old_mp);
    old_keys = flatmap_get_keys(old_mp);

    res = make_flatmap(hp);
    mp = (flatmap_t *)hp;
    hp += MAP_HEADER_FLATMAP_SZ;
    mp->thing_word = MAP_HEADER_FLATMAP;
    mp->size = num_old;
    mp->keys = old_mp->keys;

    /* Get array of key/value pairs to be updated */
    GET_TERM(*new_p, new_key);

    /* Update all values */
    for (i = 0; i < num_old; i++) {
	if (!EQ(*old_keys, new_key)) {
	    /* Not same keys */
	    *hp++ = *old_vals;
	} else {
	    GET_TERM(new_p[1], *hp);
	    hp++;
	    n--;
	    if (n == 0) {
		/*
		 * All updates done. Copy remaining values
		 * and return the result.
		 */
		for (i++, old_vals++; i < num_old; i++) {
		    *hp++ = *old_vals++;
		}
		ASSERT(hp == p->htop + need);
		p->htop = hp;
		return res;
	    } else {
		new_p += 2;
		GET_TERM(*new_p, new_key);
	    }
	}
	old_vals++, old_keys++;
    }

    /*
     * Updates left. That means that at least one the keys in the
     * update list did not previously exist.
     */
    ASSERT(hp == p->htop + need);
    p->freason = BADKEY;
    p->fvalue = new_key;
    return THE_NON_VALUE;
}
#undef GET_TERM

int catchlevel(Process *p)
{
    return p->catches;
}

/*
 * Check if the given function is built-in (i.e. a BIF implemented in C).
 *
 * Returns 0 if not built-in, and a non-zero value if built-in.
 */

int
erts_is_builtin(Eterm Mod, Eterm Name, int arity)
{
    Export e;
    Export* ep;

    e.code[0] = Mod;
    e.code[1] = Name;
    e.code[2] = arity;

    if ((ep = export_get(&e)) == NULL) {
	return 0;
    }
    return ep->addressv[erts_active_code_ix()] == ep->code+3
	&& (ep->code[3] == (BeamInstr) em_apply_bif);
}


/*
 * Return the current number of reductions for the given process.
 * To get the total number of reductions, p->reds must be added.
 */

Uint
erts_current_reductions(Process *current, Process *p)
{
    if (current != p) {
	return 0;
    } else if (current->fcalls < 0 && ERTS_PROC_GET_SAVED_CALLS_BUF(current)) {
	return -current->fcalls;
    } else {
	return REDS_IN(current) - current->fcalls;
    }
}

int
erts_beam_jump_table(void)
{
#if defined(NO_JUMP_TABLE)
    return 0;
#else
# 7152 "beam/beam_emu.c"
    return 1;
#endif
# 7154 "beam/beam_emu.c"
}