From d5b13b06d739c0b9756223eb4aefb4560a1d0ea3 Mon Sep 17 00:00:00 2001 From: Daniele Bagni Date: Tue, 8 Sep 2026 09:46:45 +0200 Subject: [PATCH 1/2] Use ROCm clang for Ghost unified-memory examples Signed-off-by: Daniele Bagni --- tests/mpi_ghost_exchange_ver1.sh | 5 +++++ tests/mpi_ghost_exchange_ver2.sh | 5 +++++ tests/mpi_ghost_exchange_ver3.sh | 5 +++++ tests/mpi_ghost_exchange_ver4.sh | 5 +++++ tests/mpi_ghost_exchange_ver5.sh | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/tests/mpi_ghost_exchange_ver1.sh b/tests/mpi_ghost_exchange_ver1.sh index 53b7c81f..e86990c6 100755 --- a/tests/mpi_ghost_exchange_ver1.sh +++ b/tests/mpi_ghost_exchange_ver1.sh @@ -36,6 +36,11 @@ BUILD_DIR=$(mktemp -d) trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} +if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" +fi + cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver2.sh b/tests/mpi_ghost_exchange_ver2.sh index 0887392e..c986d544 100755 --- a/tests/mpi_ghost_exchange_ver2.sh +++ b/tests/mpi_ghost_exchange_ver2.sh @@ -41,6 +41,11 @@ BUILD_DIR=$(mktemp -d) trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} +if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" +fi + cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver3.sh b/tests/mpi_ghost_exchange_ver3.sh index 87b3166d..b370b4f4 100755 --- a/tests/mpi_ghost_exchange_ver3.sh +++ b/tests/mpi_ghost_exchange_ver3.sh @@ -54,6 +54,11 @@ else cd ${BUILD_DIR} echo "cmake ${SRC_DIR}" + if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi + cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver4.sh b/tests/mpi_ghost_exchange_ver4.sh index eab856c8..dc3c7763 100755 --- a/tests/mpi_ghost_exchange_ver4.sh +++ b/tests/mpi_ghost_exchange_ver4.sh @@ -41,6 +41,11 @@ else trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} + if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi + cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver5.sh b/tests/mpi_ghost_exchange_ver5.sh index f17a95a7..8e3c7702 100755 --- a/tests/mpi_ghost_exchange_ver5.sh +++ b/tests/mpi_ghost_exchange_ver5.sh @@ -43,6 +43,11 @@ else trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} + if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi + cmake ${SRC_DIR} make From 61a35e7e75d3f6b3b042bc5b90ca69ecde53eb18 Mon Sep 17 00:00:00 2001 From: Daniele Bagni <30289343+dannybaths@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:55:33 +0200 Subject: [PATCH 2/2] Narrow the ROCm compiler fallback to the wrapper-absent case Ghost Ver1-Ver5 previously exported the ROCm amdclang++ whenever it existed, which also overrode the Cray wrapper choice made by the CRAYPE_VERSION / /etc/cray-release branch above. On a Cray PE login that is a regression: measured on an MI300A node with ROCm 7.14, the stock scripts pass all five requested ghost CTests through the CC wrapper (CrayClang), while the unconditional override loses Ver3, Ver4 and Ver5. The failure the previous commit addressed happens in the other state: /etc/cray-release exists on a Cray system even when no PrgEnv module puts the wrappers on PATH, so the branch above exports an empty CXX and CMake falls back to /usr/bin/c++, which does not implement the OpenMP requires unified_shared_memory directive. In that state all five tests fail unpatched and pass with the ROCm compilers. Fill in the ROCm compilers only when CXX is empty or does not resolve, so a working wrapper choice or an explicit user setting is left alone. --- tests/mpi_ghost_exchange_ver1.sh | 17 ++++++++++++----- tests/mpi_ghost_exchange_ver2.sh | 17 ++++++++++++----- tests/mpi_ghost_exchange_ver3.sh | 17 ++++++++++++----- tests/mpi_ghost_exchange_ver4.sh | 17 ++++++++++++----- tests/mpi_ghost_exchange_ver5.sh | 17 ++++++++++++----- 5 files changed, 60 insertions(+), 25 deletions(-) diff --git a/tests/mpi_ghost_exchange_ver1.sh b/tests/mpi_ghost_exchange_ver1.sh index e86990c6..99ec749f 100755 --- a/tests/mpi_ghost_exchange_ver1.sh +++ b/tests/mpi_ghost_exchange_ver1.sh @@ -26,6 +26,18 @@ else module load openmpi fi +# /etc/cray-release exists on a Cray system even when no PrgEnv module has put +# the compiler wrappers on PATH. The branch above then exports an empty CXX and +# CMake falls back to /usr/bin/c++, which does not implement +# "#pragma omp requires unified_shared_memory". Use the ROCm compilers only in +# that case, so a working wrapper or an explicit user choice is left alone. +if [ -z "${CXX:-}" ] || ! command -v "${CXX}" >/dev/null 2>&1; then + if [ -x "${ROCM_PATH:-}/bin/amdclang++" ] && [ -x "${ROCM_PATH:-}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi +fi + REPO_DIR="$(dirname "$(dirname "$(readlink -fm "$0")")")" cd ${REPO_DIR}/MPI-examples/GhostExchange/GhostExchange_ArrayAssign @@ -36,11 +48,6 @@ BUILD_DIR=$(mktemp -d) trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} -if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then - export CXX="${ROCM_PATH}/bin/amdclang++" - export CC="${ROCM_PATH}/bin/amdclang" -fi - cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver2.sh b/tests/mpi_ghost_exchange_ver2.sh index c986d544..db7e9531 100755 --- a/tests/mpi_ghost_exchange_ver2.sh +++ b/tests/mpi_ghost_exchange_ver2.sh @@ -24,6 +24,18 @@ else module load openmpi fi +# /etc/cray-release exists on a Cray system even when no PrgEnv module has put +# the compiler wrappers on PATH. The branch above then exports an empty CXX and +# CMake falls back to /usr/bin/c++, which does not implement +# "#pragma omp requires unified_shared_memory". Use the ROCm compilers only in +# that case, so a working wrapper or an explicit user choice is left alone. +if [ -z "${CXX:-}" ] || ! command -v "${CXX}" >/dev/null 2>&1; then + if [ -x "${ROCM_PATH:-}/bin/amdclang++" ] && [ -x "${ROCM_PATH:-}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi +fi + export HSA_XNACK=1 if ! command -v rocprof-sys-instrument >/dev/null 2>&1; then @@ -41,11 +53,6 @@ BUILD_DIR=$(mktemp -d) trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} -if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then - export CXX="${ROCM_PATH}/bin/amdclang++" - export CC="${ROCM_PATH}/bin/amdclang" -fi - cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver3.sh b/tests/mpi_ghost_exchange_ver3.sh index b370b4f4..b913e53a 100755 --- a/tests/mpi_ghost_exchange_ver3.sh +++ b/tests/mpi_ghost_exchange_ver3.sh @@ -24,6 +24,18 @@ else module load openmpi fi +# /etc/cray-release exists on a Cray system even when no PrgEnv module has put +# the compiler wrappers on PATH. The branch above then exports an empty CXX and +# CMake falls back to /usr/bin/c++, which does not implement +# "#pragma omp requires unified_shared_memory". Use the ROCm compilers only in +# that case, so a working wrapper or an explicit user choice is left alone. +if [ -z "${CXX:-}" ] || ! command -v "${CXX}" >/dev/null 2>&1; then + if [ -x "${ROCM_PATH:-}/bin/amdclang++" ] && [ -x "${ROCM_PATH:-}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi +fi + XNACK_COUNT=`rocminfo | grep xnack | wc -l` if [ ${XNACK_COUNT} -lt 1 ]; then echo "Skip" @@ -54,11 +66,6 @@ else cd ${BUILD_DIR} echo "cmake ${SRC_DIR}" - if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then - export CXX="${ROCM_PATH}/bin/amdclang++" - export CC="${ROCM_PATH}/bin/amdclang" - fi - cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver4.sh b/tests/mpi_ghost_exchange_ver4.sh index dc3c7763..4be7ef3c 100755 --- a/tests/mpi_ghost_exchange_ver4.sh +++ b/tests/mpi_ghost_exchange_ver4.sh @@ -24,6 +24,18 @@ else module load openmpi fi +# /etc/cray-release exists on a Cray system even when no PrgEnv module has put +# the compiler wrappers on PATH. The branch above then exports an empty CXX and +# CMake falls back to /usr/bin/c++, which does not implement +# "#pragma omp requires unified_shared_memory". Use the ROCm compilers only in +# that case, so a working wrapper or an explicit user choice is left alone. +if [ -z "${CXX:-}" ] || ! command -v "${CXX}" >/dev/null 2>&1; then + if [ -x "${ROCM_PATH:-}/bin/amdclang++" ] && [ -x "${ROCM_PATH:-}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi +fi + XNACK_COUNT=`rocminfo | grep xnack | wc -l` if [ ${XNACK_COUNT} -lt 1 ]; then echo "Skip" @@ -41,11 +53,6 @@ else trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} - if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then - export CXX="${ROCM_PATH}/bin/amdclang++" - export CC="${ROCM_PATH}/bin/amdclang" - fi - cmake ${SRC_DIR} make diff --git a/tests/mpi_ghost_exchange_ver5.sh b/tests/mpi_ghost_exchange_ver5.sh index 8e3c7702..086ad427 100755 --- a/tests/mpi_ghost_exchange_ver5.sh +++ b/tests/mpi_ghost_exchange_ver5.sh @@ -24,6 +24,18 @@ else module load openmpi fi +# /etc/cray-release exists on a Cray system even when no PrgEnv module has put +# the compiler wrappers on PATH. The branch above then exports an empty CXX and +# CMake falls back to /usr/bin/c++, which does not implement +# "#pragma omp requires unified_shared_memory". Use the ROCm compilers only in +# that case, so a working wrapper or an explicit user choice is left alone. +if [ -z "${CXX:-}" ] || ! command -v "${CXX}" >/dev/null 2>&1; then + if [ -x "${ROCM_PATH:-}/bin/amdclang++" ] && [ -x "${ROCM_PATH:-}/bin/amdclang" ]; then + export CXX="${ROCM_PATH}/bin/amdclang++" + export CC="${ROCM_PATH}/bin/amdclang" + fi +fi + XNACK_COUNT=`rocminfo | grep xnack | wc -l` if [ ${XNACK_COUNT} -lt 1 ]; then echo "Skip" @@ -43,11 +55,6 @@ else trap "rm -rf ${BUILD_DIR}" EXIT cd ${BUILD_DIR} - if [ -n "${ROCM_PATH:-}" ] && [ -x "${ROCM_PATH}/bin/amdclang++" ] && [ -x "${ROCM_PATH}/bin/amdclang" ]; then - export CXX="${ROCM_PATH}/bin/amdclang++" - export CC="${ROCM_PATH}/bin/amdclang" - fi - cmake ${SRC_DIR} make