diff --git a/CHANGES b/CHANGES index 6a9eee18f5..26cdfad324 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,6 @@ +6097. [port] Improve support for yield / pause instructions in spin + loops on AArch64 platforms. [GL !7469] + 6096. [bug] Fix RPZ reference counting error on shutdown in dns__rpz_timer_cb(). [GL #3866] diff --git a/configure.ac b/configure.ac index c6106cd5cd..4aa1761b26 100644 --- a/configure.ac +++ b/configure.ac @@ -359,8 +359,8 @@ AC_C_VOLATILE # Check for yield support on ARM processors # AS_CASE([$host], - [arm*], - [AC_MSG_CHECKING([for yield instruction support]) + [arm*|aarch64*], + [AC_MSG_CHECKING([for ARM yield instruction support]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[]], [[__asm__ __volatile__ ("yield")]])], diff --git a/lib/isc/Makefile.am b/lib/isc/Makefile.am index 8201e71d6a..b532587cf2 100644 --- a/lib/isc/Makefile.am +++ b/lib/isc/Makefile.am @@ -67,6 +67,7 @@ libisc_la_HEADERS = \ include/isc/once.h \ include/isc/os.h \ include/isc/parseint.h \ + include/isc/pause.h \ include/isc/portset.h \ include/isc/print.h \ include/isc/quota.h \ diff --git a/lib/isc/include/isc/pause.h b/lib/isc/include/isc/pause.h new file mode 100644 index 0000000000..8d7fa57682 --- /dev/null +++ b/lib/isc/include/isc/pause.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) Internet Systems Consortium, Inc. ("ISC") + * + * SPDX-License-Identifier: MPL-2.0 + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, you can obtain one at https://mozilla.org/MPL/2.0/. + * + * See the COPYRIGHT file distributed with this work for additional + * information regarding copyright ownership. + */ + +#pragma once + +#if defined(__x86_64__) +#include +#define isc_pause() _mm_pause() +#elif defined(__i386__) +#define isc_pause() __asm__ __volatile__("rep; nop") +#elif defined(__ia64__) +#define isc_pause() __asm__ __volatile__("hint @pause") +#elif defined(__aarch64__) +#define isc_pause() __asm__ __volatile__("isb") +#elif defined(__arm__) && HAVE_ARM_YIELD +#define isc_pause() __asm__ __volatile__("yield") +#elif defined(sun) && (defined(__sparc) || defined(__sparc__)) +#include +#define isc_pause() smt_pause() +#elif (defined(__sparc) || defined(__sparc__)) && HAVE_SPARC_PAUSE +#define isc_pause() __asm__ __volatile__("pause") +#elif defined(__ppc__) || defined(_ARCH_PPC) || defined(_ARCH_PWR) || \ + defined(_ARCH_PWR2) || defined(_POWER) +#define isc_pause() __asm__ volatile("or 27,27,27") +#else +#define isc_pause() sched_yield() +#endif + +#define isc_pause_n(iters) \ + for (size_t __pause = 0; __pause < iters; __pause++) { \ + isc_pause(); \ + } diff --git a/lib/isc/rwlock.c b/lib/isc/rwlock.c index d00ec7655f..62bdbf1a62 100644 --- a/lib/isc/rwlock.c +++ b/lib/isc/rwlock.c @@ -17,12 +17,9 @@ #include #include -#if defined(sun) && (defined(__sparc) || defined(__sparc__)) -#include /* for smt_pause(3c) */ -#endif /* if defined(sun) && (defined(__sparc) || defined(__sparc__)) */ - #include #include +#include #include #include #include @@ -42,29 +39,6 @@ #define RWLOCK_MAX_ADAPTIVE_COUNT 100 #endif /* ifndef RWLOCK_MAX_ADAPTIVE_COUNT */ -#if defined(_MSC_VER) -#include -#define isc_rwlock_pause() YieldProcessor() -#elif defined(__x86_64__) -#include -#define isc_rwlock_pause() _mm_pause() -#elif defined(__i386__) -#define isc_rwlock_pause() __asm__ __volatile__("rep; nop") -#elif defined(__ia64__) -#define isc_rwlock_pause() __asm__ __volatile__("hint @pause") -#elif defined(__arm__) && HAVE_ARM_YIELD -#define isc_rwlock_pause() __asm__ __volatile__("yield") -#elif defined(sun) && (defined(__sparc) || defined(__sparc__)) -#define isc_rwlock_pause() smt_pause() -#elif (defined(__sparc) || defined(__sparc__)) && HAVE_SPARC_PAUSE -#define isc_rwlock_pause() __asm__ __volatile__("pause") -#elif defined(__ppc__) || defined(_ARCH_PPC) || defined(_ARCH_PWR) || \ - defined(_ARCH_PWR2) || defined(_POWER) -#define isc_rwlock_pause() __asm__ volatile("or 27,27,27") -#else /* if defined(_MSC_VER) */ -#define isc_rwlock_pause() -#endif /* if defined(_MSC_VER) */ - #ifdef ISC_RWLOCK_TRACE #include /* Required for fprintf/stderr. */ @@ -331,7 +305,7 @@ isc__rwlock_lock(isc__rwlock_t *rwl, isc_rwlocktype_t type) { rwlock_lock(rwl, type); break; } - isc_rwlock_pause(); + isc_pause(); } while (isc_rwlock_trylock(rwl, type) != ISC_R_SUCCESS); atomic_fetch_add_release(&rwl->spins, (cnt - spins) / 8);