kokkos
/
Makefile.kokkos

# Default settings common options.

KOKKOS_VERSION_MAJOR = 4
KOKKOS_VERSION_MINOR = 3
KOKKOS_VERSION_PATCH = 99
KOKKOS_VERSION = $(shell echo $(KOKKOS_VERSION_MAJOR)*10000+$(KOKKOS_VERSION_MINOR)*100+$(KOKKOS_VERSION_PATCH) | bc)

# Options: Cuda,HIP,SYCL,OpenMPTarget,OpenMP,Threads,Serial
#KOKKOS_DEVICES ?= "OpenMP"
KOKKOS_DEVICES ?= "Threads"
# Options:
# Intel:    KNC,KNL,SNB,HSW,BDW,SKL,SKX,ICL,ICX,SPR
# NVIDIA:   Kepler,Kepler30,Kepler32,Kepler35,Kepler37,Maxwell,Maxwell50,Maxwell52,Maxwell53,Pascal60,Pascal61,Volta70,Volta72,Turing75,Ampere80,Ampere86,Ada89,Hopper90
# ARM:      ARMv80,ARMv81,ARMv8-ThunderX,ARMv8-TX2,A64FX
# IBM:      Power8,Power9
# AMD-GPUS: AMD_GFX906,AMD_GFX908,AMD_GFX90A,AMD_GFX940,AMD_GFX942,AMD_GFX1030,AMD_GFX1100
# AMD-CPUS: AMDAVX,Zen,Zen2,Zen3
# Intel-GPUs: Intel_Gen,Intel_Gen9,Intel_Gen11,Intel_Gen12LP,Intel_DG1,Intel_XeHP,Intel_PVC
KOKKOS_ARCH ?= ""
# Options: yes,no
KOKKOS_DEBUG ?= "no"
# Options: hwloc
KOKKOS_USE_TPLS ?= ""
# Options: c++17,c++1z,c++20,c++2a,c++23,c++2b
KOKKOS_CXX_STANDARD ?= "c++17"
# Options: aggressive_vectorization,disable_profiling,enable_large_mem_tests,disable_complex_align,disable_deprecated_code,enable_deprecation_warnings
KOKKOS_OPTIONS ?= ""
KOKKOS_CMAKE ?= "no"
KOKKOS_TRIBITS ?= "no"
KOKKOS_STANDALONE_CMAKE ?= "no"

# Default settings specific options.
# Options: force_uvm,use_ldg,rdc,enable_lambda,enable_constexpr,disable_malloc_async
KOKKOS_CUDA_OPTIONS ?= ""

# Options: rdc
KOKKOS_HIP_OPTIONS ?= ""

# Default settings specific options.
# Options: enable_async_dispatch
KOKKOS_HPX_OPTIONS ?= ""

# Helper functions for conversion to upper case
uppercase_TABLE:=a,A b,B c,C d,D e,E f,F g,G h,H i,I j,J k,K l,L m,M n,N o,O p,P q,Q r,R s,S t,T u,U v,V w,W x,X y,Y z,Z
uppercase_internal=$(if $1,$$(subst $(firstword $1),$(call uppercase_internal,$(wordlist 2,$(words $1),$1),$2)),$2)
uppercase=$(eval uppercase_RESULT:=$(call uppercase_internal,$(uppercase_TABLE),$1))$(uppercase_RESULT)
# Return a 1 if a string contains a substring and 0 if not
# Note the search string should be without '"'
# Example: $(call kokkos_has_string,"hwloc,libdl",hwloc)
#   Will return a 1
kokkos_has_string=$(if $(findstring $(call uppercase,$2),$(call uppercase,$1)),1,0)
# Returns 1 if the path exists, 0 otherwise
# Example: $(call kokkos_path_exists,/path/to/file)
#   Will return a 1 if /path/to/file exists
kokkos_path_exists=$(if $(wildcard $1),1,0)

# Check for general settings

KOKKOS_INTERNAL_ENABLE_DEBUG := $(call kokkos_has_string,$(KOKKOS_DEBUG),yes)
KOKKOS_INTERNAL_ENABLE_CXX17 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++17)
KOKKOS_INTERNAL_ENABLE_CXX1Z := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++1z)
KOKKOS_INTERNAL_ENABLE_CXX20 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++20)
KOKKOS_INTERNAL_ENABLE_CXX2A := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++2a)
KOKKOS_INTERNAL_ENABLE_CXX23 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++23)
KOKKOS_INTERNAL_ENABLE_CXX2B := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++2b)
KOKKOS_INTERNAL_ENABLE_CXX26 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++26)
KOKKOS_INTERNAL_ENABLE_CXX2C := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++2c)

# Check for external libraries.
KOKKOS_INTERNAL_USE_HWLOC := $(call kokkos_has_string,$(KOKKOS_USE_TPLS),hwloc)

# Check for advanced settings.
KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),compiler_warnings)
KOKKOS_INTERNAL_AGGRESSIVE_VECTORIZATION := $(call kokkos_has_string,$(KOKKOS_OPTIONS),aggressive_vectorization)
KOKKOS_INTERNAL_ENABLE_TUNING := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_tuning)
KOKKOS_INTERNAL_DISABLE_COMPLEX_ALIGN := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_complex_align)
KOKKOS_INTERNAL_DISABLE_DUALVIEW_MODIFY_CHECK := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_dualview_modify_check)
KOKKOS_INTERNAL_ENABLE_LARGE_MEM_TESTS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_large_mem_tests)
# deprecated
KOKKOS_INTERNAL_CUDA_USE_LDG := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),use_ldg)
KOKKOS_INTERNAL_CUDA_USE_UVM := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),force_uvm)
KOKKOS_INTERNAL_CUDA_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),rdc)
KOKKOS_INTERNAL_CUDA_USE_LAMBDA := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_lambda)
KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_constexpr)
KOKKOS_INTERNAL_CUDA_DISABLE_MALLOC_ASYNC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),disable_malloc_async)
KOKKOS_INTERNAL_HPX_ENABLE_ASYNC_DISPATCH := $(call kokkos_has_string,$(KOKKOS_HPX_OPTIONS),enable_async_dispatch)
# deprecated
KOKKOS_INTERNAL_ENABLE_DESUL_ATOMICS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_desul_atomics)
# deprecated
KOKKOS_INTERNAL_DISABLE_DESUL_ATOMICS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_desul_atomics)
KOKKOS_INTERNAL_DISABLE_BUNDLED_MDSPAN := $(call kokkos_has_string,$(KOKKOS_OPTIONS),impl_disable_bundled_mdspan)
KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_deprecated_code)
KOKKOS_INTERNAL_ENABLE_DEPRECATION_WARNINGS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_deprecation_warnings)

KOKKOS_INTERNAL_HIP_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_HIP_OPTIONS),rdc)

# Check for Kokkos Host Execution Spaces one of which must be on.
KOKKOS_INTERNAL_USE_OPENMP := $(call kokkos_has_string,$(subst OpenMPTarget,,$(KOKKOS_DEVICES)),OpenMP)
KOKKOS_INTERNAL_USE_THREADS := $(call kokkos_has_string,$(KOKKOS_DEVICES),Threads)
KOKKOS_INTERNAL_USE_HPX := $(call kokkos_has_string,$(KOKKOS_DEVICES),HPX)
KOKKOS_INTERNAL_USE_SERIAL := $(call kokkos_has_string,$(KOKKOS_DEVICES),Serial)

ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 0)
  ifeq ($(KOKKOS_INTERNAL_USE_THREADS), 0)
    ifeq ($(KOKKOS_INTERNAL_USE_HPX), 0)
      KOKKOS_INTERNAL_USE_SERIAL := 1
    endif
  endif
endif

# Check for other Execution Spaces.
KOKKOS_INTERNAL_USE_CUDA := $(call kokkos_has_string,$(KOKKOS_DEVICES),Cuda)
KOKKOS_INTERNAL_USE_HIP := $(call kokkos_has_string,$(KOKKOS_DEVICES),HIP)
KOKKOS_INTERNAL_USE_SYCL := $(call kokkos_has_string,$(KOKKOS_DEVICES),SYCL)
KOKKOS_INTERNAL_USE_OPENMPTARGET := $(call kokkos_has_string,$(KOKKOS_DEVICES),OpenMPTarget)
KOKKOS_INTERNAL_USE_OPENACC := $(call kokkos_has_string,$(KOKKOS_DEVICES),OpenACC)

KOKKOS_DEVICELIST =
ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1)
  KOKKOS_DEVICELIST += Serial
endif
ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1)
  KOKKOS_DEVICELIST += OpenMP
endif
ifeq ($(KOKKOS_INTERNAL_USE_THREADS), 1)
  KOKKOS_DEVICELIST += Threads
endif
ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1)
  KOKKOS_DEVICELIST += HPX
endif
ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  KOKKOS_DEVICELIST += Cuda
endif
ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
  KOKKOS_DEVICELIST += HIP
endif
ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
  KOKKOS_DEVICELIST += SYCL
endif
ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  KOKKOS_DEVICELIST += OPENMPTARGET
endif
ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1)
  KOKKOS_DEVICELIST += OpenACC
endif

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc)
  ifeq ($(origin CUDA_PATH), undefined)
    CUDA_PATH = $(KOKKOS_INTERNAL_NVCC_PATH:/bin/nvcc=)
  endif
  ifeq ($(CUDA_PATH),)
    CUDA_PATH = $(KOKKOS_INTERNAL_NVCC_PATH:/bin/nvcc=)
  endif
  KOKKOS_INTERNAL_COMPILER_NVCC_VERSION := $(shell nvcc --version 2>&1 | grep release | cut -d' ' -f5 | cut -d',' -f1 | tr -d .)
endif

# Check OS.
KOKKOS_OS                      := $(strip $(shell uname -s))
KOKKOS_INTERNAL_OS_CYGWIN      := $(call kokkos_has_string,$(KOKKOS_OS),CYGWIN)
KOKKOS_INTERNAL_OS_LINUX       := $(call kokkos_has_string,$(KOKKOS_OS),Linux)
KOKKOS_INTERNAL_OS_DARWIN      := $(call kokkos_has_string,$(KOKKOS_OS),Darwin)

# Check compiler.
KOKKOS_CXX_VERSION                   := $(strip $(shell $(CXX) --version       2>&1))
KOKKOS_INTERNAL_COMPILER_INTEL       := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Intel Corporation)
KOKKOS_INTERNAL_COMPILER_CRAY        := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep -c "CC-"))
KOKKOS_INTERNAL_COMPILER_NVCC        := $(strip $(shell echo "$(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep -c nvcc)>0" | bc))
KOKKOS_INTERNAL_COMPILER_NVHPC       := $(strip $(shell $(CXX) --version 2>&1 | grep -c "nvc++"))
KOKKOS_INTERNAL_COMPILER_CLANG       := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),clang)
KOKKOS_INTERNAL_COMPILER_CRAY_CLANG  := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep -c "clang++"))
KOKKOS_INTERNAL_COMPILER_INTEL_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),oneAPI)
KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple clang)
KOKKOS_INTERNAL_COMPILER_HCC         := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC)
KOKKOS_INTERNAL_COMPILER_GCC         := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),GCC)
# TODO fujitsu can emulate gcc or clang. Only clang mode works at the moment.
KOKKOS_INTERNAL_COMPILER_FUJITSU     := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),FUJITSU)

# Check Host Compiler if using NVCC through nvcc_wrapper
ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
  KOKKOS_INTERNAL_COMPILER_NVCC_WRAPPER := $(strip $(shell echo $(CXX) | grep -c nvcc_wrapper))
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC_WRAPPER), 1)

    KOKKOS_CXX_HOST_VERSION             := $(strip $(shell $(CXX) $(CXXFLAGS) --host-version       2>&1))
    KOKKOS_INTERNAL_COMPILER_INTEL  := $(call kokkos_has_string,$(KOKKOS_CXX_HOST_VERSION),Intel Corporation)
    KOKKOS_INTERNAL_COMPILER_CLANG  := $(call kokkos_has_string,$(KOKKOS_CXX_HOST_VERSION),clang)
  endif
endif

ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 2)
  KOKKOS_INTERNAL_COMPILER_CLANG = 1
endif

# Apple Clang passes both clang and apple clang tests, so turn off clang.
ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1)
  KOKKOS_INTERNAL_COMPILER_CLANG = 0
endif
# AMD HCC passes both clang and hcc test so turn off clang
ifeq ($(KOKKOS_INTERNAL_COMPILER_HCC), 1)
  KOKKOS_INTENAL_COMPILER_CLANG = 0
endif
# Fujitsu passes also as clang and gcc respectively
ifeq ($(KOKKOS_INTERNAL_COMPILER_FUJITSU), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_GCC), 1)
    # TODO  handle gcc flags and workaround for bug?
    # fujitsu (gcc mode) is bugged, see https://github.com/kokkos/kokkos/issues/4730
    $(warning Warning: ${CXX} in Trad Mode '-Nnoclang' (default) is not recommended. Use 'CXX = ${CXX} -Nclang' instead.)
    # HACK since fujitsu only accepts some gcc flags, disable gcc here?
    # KOKKOS_INTERNAL_COMPILER_GCC = 0
  endif
  # TODO handle clang flags
  # warnings: works fine as is
  # openmp: handled
  #KOKKOS_INTERNAL_COMPILER_CLANG = 0
endif

ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
  # TODO empty variable if fujitsu (clang mode) passes as clang
  KOKKOS_INTERNAL_COMPILER_CLANG_VERSION := $(shell $(CXX) --version | grep version | cut -d ' ' -f3 | tr -d '.')

  ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
    ifeq ($(shell test $(KOKKOS_INTERNAL_COMPILER_CLANG_VERSION) -lt 400; echo $$?),0)
      $(error Compiling Cuda code directly with Clang requires version 4.0.0 or higher)
    endif

    KOKKOS_INTERNAL_CUDA_USE_LAMBDA := 1
  endif
endif

# Set compiler warnings flags.
ifeq ($(KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1)
    KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    # TODO check if cray accepts GNU style warnings
    KOKKOS_INTERNAL_COMPILER_WARNINGS =
  else
    #gcc
    KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wunused-parameter -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wignored-qualifiers -Wempty-body -Wclobbered -Wuninitialized
  endif
else
  KOKKOS_INTERNAL_COMPILER_WARNINGS =
endif

# Set OpenMP flags.
ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY_CLANG), 1)
    KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_FUJITSU), 1)
    # fujitsu (clang mode) fails with `=libomp`
    KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp
  else
    KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp=libomp
  endif
else
  ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1)
    KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp=libomp
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    # OpenMP is turned on by default in Cray compiler environment.
    KOKKOS_INTERNAL_OPENMP_FLAG :=
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL_CLANG), 1)
    KOKKOS_INTERNAL_OPENMP_FLAG := -fiopenmp
  else
    KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL_CLANG), 1)
    KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -fiopenmp -Wno-openmp-mapping
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1)
    KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -mp=gpu
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 0)
    #Assume GCC
    KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -fopenmp -foffload=nvptx-none
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1)
  # Set OpenACC flags.
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1)
    KOKKOS_INTERNAL_OPENACC_FLAG := -acc
  else
    $(error Makefile.kokkos: OpenACC is enabled but the compiler must be NVHPC (got version string $(KOKKOS_CXX_VERSION)))
  endif
endif

# Set C++ version flags.
ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  KOKKOS_INTERNAL_CXX17_FLAG := -hstd=c++17
  KOKKOS_INTERNAL_CXX1Z_FLAG := -hstd=c++1z
  KOKKOS_INTERNAL_CXX20_FLAG := -hstd=c++20
  KOKKOS_INTERNAL_CXX2A_FLAG := -hstd=c++2a
  KOKKOS_INTERNAL_CXX23_FLAG := -hstd=c++23
  KOKKOS_INTERNAL_CXX2A_FLAG := -hstd=c++2b
else
  KOKKOS_INTERNAL_CXX17_FLAG := -std=c++17
  KOKKOS_INTERNAL_CXX1Z_FLAG := -std=c++1z
  KOKKOS_INTERNAL_CXX20_FLAG := -std=c++20
  KOKKOS_INTERNAL_CXX2A_FLAG := -std=c++2a
  KOKKOS_INTERNAL_CXX23_FLAG := -std=c++23
  KOKKOS_INTERNAL_CXX2B_FLAG := -std=c++2b
endif

# Check for Kokkos Architecture settings.

# Intel based.
KOKKOS_INTERNAL_USE_ARCH_KNC := $(call kokkos_has_string,$(KOKKOS_ARCH),KNC)
KOKKOS_INTERNAL_USE_ARCH_SNB := $(call kokkos_has_string,$(KOKKOS_ARCH),SNB)
KOKKOS_INTERNAL_USE_ARCH_HSW := $(call kokkos_has_string,$(KOKKOS_ARCH),HSW)
KOKKOS_INTERNAL_USE_ARCH_BDW := $(call kokkos_has_string,$(KOKKOS_ARCH),BDW)
KOKKOS_INTERNAL_USE_ARCH_SKL := $(call kokkos_has_string,$(KOKKOS_ARCH),SKL)
KOKKOS_INTERNAL_USE_ARCH_SKX := $(call kokkos_has_string,$(KOKKOS_ARCH),SKX)
KOKKOS_INTERNAL_USE_ARCH_KNL := $(call kokkos_has_string,$(KOKKOS_ARCH),KNL)
KOKKOS_INTERNAL_USE_ARCH_ICL := $(call kokkos_has_string,$(KOKKOS_ARCH),ICL)
KOKKOS_INTERNAL_USE_ARCH_ICX := $(call kokkos_has_string,$(KOKKOS_ARCH),ICX)
KOKKOS_INTERNAL_USE_ARCH_SPR := $(call kokkos_has_string,$(KOKKOS_ARCH),SPR)

# Traditionally, we supported, e.g., IntelGen9 instead of Intel_Gen9. The latter
# matches the CMake option but we also accept the former for backward-compatibility.
KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9 := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelGen9)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9 := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_Gen9)
endif
KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN11 := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelGen11)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN11), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN11 := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_Gen11)
endif
KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN12LP := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelGen12LP)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN12LP), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN12LP := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_Gen12LP)
endif
KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelGen9)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9 := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_Gen9)
endif
KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN_SET := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9)  \
                                                     + $(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN11) \
                                                     + $(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN12LP))
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN_SET), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelGen)
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN), 0)
    KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_Gen)
  endif
endif
KOKKOS_INTERNAL_USE_ARCH_INTEL_DG1 := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelDG1)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_DG1), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_DG1 := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_DG1)
endif
KOKKOS_INTERNAL_USE_ARCH_INTEL_XEHP := $(call kokkos_has_string,$(KOKKOS_ARCH),IntelXeHP)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_XEHP), 0)
  KOKKOS_INTERNAL_USE_ARCH_INTEL_XEHP := $(call kokkos_has_string,$(KOKKOS_ARCH),Intel_XeHP)
endif
# Traditionally the architecture was called PVC instead of Intel_PVC. This
# version makes us accept IntelPVC and Intel_PVC as well.
KOKKOS_INTERNAL_USE_ARCH_INTEL_PVC := $(call kokkos_has_string,$(KOKKOS_ARCH),PVC)

# NVIDIA based.
NVCC_WRAPPER := $(KOKKOS_PATH)/bin/nvcc_wrapper
KOKKOS_INTERNAL_USE_ARCH_KEPLER30 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler30)
KOKKOS_INTERNAL_USE_ARCH_KEPLER32 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler32)
KOKKOS_INTERNAL_USE_ARCH_KEPLER35 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler35)
KOKKOS_INTERNAL_USE_ARCH_KEPLER37 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler37)
KOKKOS_INTERNAL_USE_ARCH_MAXWELL50 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell50)
KOKKOS_INTERNAL_USE_ARCH_MAXWELL52 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell52)
KOKKOS_INTERNAL_USE_ARCH_MAXWELL53 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell53)
KOKKOS_INTERNAL_USE_ARCH_PASCAL61 := $(call kokkos_has_string,$(KOKKOS_ARCH),Pascal61)
KOKKOS_INTERNAL_USE_ARCH_PASCAL60 := $(call kokkos_has_string,$(KOKKOS_ARCH),Pascal60)
KOKKOS_INTERNAL_USE_ARCH_VOLTA70 := $(call kokkos_has_string,$(KOKKOS_ARCH),Volta70)
KOKKOS_INTERNAL_USE_ARCH_VOLTA72 := $(call kokkos_has_string,$(KOKKOS_ARCH),Volta72)
KOKKOS_INTERNAL_USE_ARCH_TURING75 := $(call kokkos_has_string,$(KOKKOS_ARCH),Turing75)
KOKKOS_INTERNAL_USE_ARCH_AMPERE80 := $(call kokkos_has_string,$(KOKKOS_ARCH),Ampere80)
KOKKOS_INTERNAL_USE_ARCH_AMPERE86 := $(call kokkos_has_string,$(KOKKOS_ARCH),Ampere86)
KOKKOS_INTERNAL_USE_ARCH_ADA89 := $(call kokkos_has_string,$(KOKKOS_ARCH),Ada89)
KOKKOS_INTERNAL_USE_ARCH_HOPPER90 := $(call kokkos_has_string,$(KOKKOS_ARCH),Hopper90)
KOKKOS_INTERNAL_USE_ARCH_NVIDIA := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KEPLER30)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER32)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER35)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER37)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50) \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52) \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53) \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_PASCAL61)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_PASCAL60)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_VOLTA70)   \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_VOLTA72)   \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_TURING75)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_AMPERE80)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_AMPERE86)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_ADA89)     \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_HOPPER90))

#SEK: This seems like a bug to me
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0)
  KOKKOS_INTERNAL_USE_ARCH_MAXWELL50 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell)
  KOKKOS_INTERNAL_USE_ARCH_KEPLER35 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler)
  KOKKOS_INTERNAL_USE_ARCH_NVIDIA := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KEPLER35)  \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50))
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc)
    CUDA_PATH ?= $(KOKKOS_INTERNAL_NVCC_PATH:/bin/nvcc=)
    ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
      KOKKOS_INTERNAL_OPENMPTARGET_FLAG := $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG) --cuda-path=$(CUDA_PATH)
    endif
  endif
endif
# ARM based.
KOKKOS_INTERNAL_USE_ARCH_ARMV80 := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv80)
KOKKOS_INTERNAL_USE_ARCH_ARMV81 := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv81)
KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv8-ThunderX)
KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX2 := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv8-TX2)
KOKKOS_INTERNAL_USE_ARCH_A64FX := $(call kokkos_has_string,$(KOKKOS_ARCH),A64FX)
KOKKOS_INTERNAL_USE_ARCH_ARM := $(strip $(shell echo $(KOKKOS_INTERNAL_USE_ARCH_ARMV80)+$(KOKKOS_INTERNAL_USE_ARCH_ARMV81)+$(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX)+$(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX2)+$(KOKKOS_INTERNAL_USE_ARCH_A64FX) | bc))

# IBM based.
KOKKOS_INTERNAL_USE_ARCH_POWER8 := $(call kokkos_has_string,$(KOKKOS_ARCH),Power8)
KOKKOS_INTERNAL_USE_ARCH_POWER9 := $(call kokkos_has_string,$(KOKKOS_ARCH),Power9)
KOKKOS_INTERNAL_USE_ARCH_IBM := $(strip $(shell echo $(KOKKOS_INTERNAL_USE_ARCH_POWER8)+$(KOKKOS_INTERNAL_USE_ARCH_POWER9) | bc))

# AMD based.
KOKKOS_INTERNAL_USE_ARCH_AMDAVX := $(call kokkos_has_string,$(KOKKOS_ARCH),AMDAVX)
KOKKOS_INTERNAL_USE_ARCH_ZEN3 := $(call kokkos_has_string,$(KOKKOS_ARCH),Zen3)
KOKKOS_INTERNAL_USE_ARCH_ZEN2 := $(call kokkos_has_string,$(KOKKOS_ARCH),Zen2)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ZEN3), 0)
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ZEN2), 0)
    KOKKOS_INTERNAL_USE_ARCH_ZEN := $(call kokkos_has_string,$(KOKKOS_ARCH),Zen)
  endif
endif

KOKKOS_INTERNAL_USE_ARCH_AMD_GFX906 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX906)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX906), 0)
  KOKKOS_INTERNAL_USE_ARCH_AMD_GFX906 := $(call kokkos_has_string,$(KOKKOS_ARCH),VEGA906)
endif
KOKKOS_INTERNAL_USE_ARCH_AMD_GFX908 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX908)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX908), 0)
  KOKKOS_INTERNAL_USE_ARCH_AMD_GFX908 := $(call kokkos_has_string,$(KOKKOS_ARCH),VEGA908)
endif
KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX90A)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A), 0)
  KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A := $(call kokkos_has_string,$(KOKKOS_ARCH),VEGA90A)
endif
KOKKOS_INTERNAL_USE_ARCH_AMD_GFX940 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX940)
KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX942)
KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX1030)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030), 0)
  KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030 := $(call kokkos_has_string,$(KOKKOS_ARCH),NAVI1030)
endif
KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX1100)
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100), 0)
  KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100 := $(call kokkos_has_string,$(KOKKOS_ARCH),NAVI1100)
endif

# Any AVX?
KOKKOS_INTERNAL_USE_ARCH_AVX        := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_SNB) + $(KOKKOS_INTERNAL_USE_ARCH_AMDAVX))
KOKKOS_INTERNAL_USE_ARCH_AVX2       := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_HSW) + $(KOKKOS_INTERNAL_USE_ARCH_BDW) + $(KOKKOS_INTERNAL_USE_ARCH_ZEN) + $(KOKKOS_INTERNAL_USE_ARCH_ZEN2) + $(KOKKOS_INTERNAL_USE_ARCH_ZEN3))
KOKKOS_INTERNAL_USE_ARCH_AVX512MIC  := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KNL))

# Incompatible flags?
KOKKOS_INTERNAL_USE_ARCH_MULTIHOST := $(strip $(shell echo "$(KOKKOS_INTERNAL_USE_ARCH_AVX)+$(KOKKOS_INTERNAL_USE_ARCH_AVX2)+$(KOKKOS_INTERNAL_USE_ARCH_AVX512MIC)+$(KOKKOS_INTERNAL_USE_ARCH_SKL)+$(KOKKOS_INTERNAL_USE_ARCH_SKX)+$(KOKKOS_INTERNAL_USE_ARCH_ICL)+$(KOKKOS_INTERNAL_USE_ARCH_ICX)+$(KOKKOS_INTERNAL_USE_ARCH_SPR)+$(KOKKOS_INTERNAL_USE_ARCH_KNC)+$(KOKKOS_INTERNAL_USE_ARCH_IBM)+$(KOKKOS_INTERNAL_USE_ARCH_ARM)>1") | bc)
KOKKOS_INTERNAL_USE_ARCH_MULTIGPU := $(strip $(shell echo "$(KOKKOS_INTERNAL_USE_ARCH_NVIDIA)>1") | bc)

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MULTIHOST), 1)
  $(error Defined Multiple Host architectures: KOKKOS_ARCH=$(KOKKOS_ARCH) )
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MULTIGPU), 1)
  $(error Defined Multiple GPU architectures: KOKKOS_ARCH=$(KOKKOS_ARCH) )
endif

# Generating the list of Flags.

KOKKOS_CPPFLAGS =
KOKKOS_LIBDIRS =
ifneq ($(KOKKOS_CMAKE), yes)
  KOKKOS_CPPFLAGS = -I./ -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src
endif
KOKKOS_TPL_INCLUDE_DIRS =
KOKKOS_TPL_LIBRARY_DIRS =
KOKKOS_TPL_LIBRARY_NAMES =

ifeq ($(KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS), 1)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_COMPILER_WARNINGS)
endif

KOKKOS_LIBS = -ldl
KOKKOS_TPL_LIBRARY_NAMES += dl
ifneq ($(KOKKOS_CMAKE), yes)
  KOKKOS_LIBDIRS = -L$(shell pwd)
  # CXXLDFLAGS is used together with CXXFLAGS in a combined compile/link command
  KOKKOS_CXXLDFLAGS = -L$(shell pwd)
endif
KOKKOS_LINK_FLAGS =
KOKKOS_SRC =
KOKKOS_HEADERS =

# Generating the KokkosCore_config.h file.

KOKKOS_INTERNAL_CONFIG_TMP=KokkosCore_config.tmp
KOKKOS_CONFIG_HEADER=KokkosCore_config.h
# Functions for generating config header file
kokkos_append_header = $(shell echo $1 >> $(KOKKOS_INTERNAL_CONFIG_TMP))

# assign hash sign to variable for compat. with make 4.3
H := \#

# Do not append first line
tmp := $(shell echo "/* ---------------------------------------------" > KokkosCore_config.tmp)
tmp := $(call kokkos_append_header,"Makefile constructed configuration:")
tmp := $(call kokkos_append_header,"----------------------------------------------*/")

tmp := $(call kokkos_append_header,'$H''if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H)')
tmp := $(call kokkos_append_header,'$H''error "Do not include $(KOKKOS_CONFIG_HEADER) directly; include Kokkos_Macros.hpp instead."')
tmp := $(call kokkos_append_header,'$H''else')
tmp := $(call kokkos_append_header,'$H''define KOKKOS_CORE_CONFIG_H')
tmp := $(call kokkos_append_header,'$H''endif')

tmp := $(call kokkos_append_header,"")
tmp := $(call kokkos_append_header,"$H""define KOKKOS_VERSION $(KOKKOS_VERSION)")
tmp := $(call kokkos_append_header,"$H""define KOKKOS_VERSION_MAJOR $(KOKKOS_VERSION_MAJOR)")
tmp := $(call kokkos_append_header,"$H""define KOKKOS_VERSION_MINOR $(KOKKOS_VERSION_MINOR)")
tmp := $(call kokkos_append_header,"$H""define KOKKOS_VERSION_PATCH $(KOKKOS_VERSION_PATCH)")
tmp := $(call kokkos_append_header,"")

tmp := $(call kokkos_append_header,"/* Execution Spaces */")

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA")
endif

ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
  tmp := $(call kokkos_append_header,'$H''define KOKKOS_ENABLE_HIP')
endif

ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
  tmp := $(call kokkos_append_header,'$H''define KOKKOS_ENABLE_SYCL')
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  KOKKOS_LIBS += -latomic
  tmp := $(call kokkos_append_header,'$H''define KOKKOS_ENABLE_OPENMPTARGET')
  ifeq ($(KOKKOS_INTERNAL_COMPILER_GCC), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_WORKAROUND_OPENMPTARGET_GCC")
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_OPENACC")
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1)
  tmp := $(call kokkos_append_header,'$H''define KOKKOS_ENABLE_OPENMP')
endif

ifeq ($(KOKKOS_INTERNAL_USE_THREADS), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_THREADS")
endif

ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_HPX")
endif

ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_SERIAL")
endif

#only add the c++ standard flags if this is not CMake
tmp := $(call kokkos_append_header,"/* General Settings */")
ifneq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_DEPRECATED_CODE_4")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_DEPRECATION_WARNINGS), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_DEPRECATION_WARNINGS")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX17), 1)
ifneq ($(KOKKOS_STANDALONE_CMAKE), yes)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX17_FLAG)
endif
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX17")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX1Z), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX1Z_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX17")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX20), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX20_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX20")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX2A), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX2A_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX20")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX23), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX23_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX23")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX2B), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX2B_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX23")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX26), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX26_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX26")
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX2C), 1)
  #I cannot make CMake add this in a good way - so add it here
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX2C_FLAG)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CXX26")
endif

ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_CXXFLAGS += -lineinfo
  endif

  KOKKOS_CXXFLAGS += -g
  KOKKOS_LDFLAGS += -g
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_DEBUG")
  ifeq ($(KOKKOS_INTERNAL_DISABLE_DUALVIEW_MODIFY_CHECK), 0)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK")
  endif
endif
ifeq ($(KOKKOS_INTERNAL_DISABLE_COMPLEX_ALIGN), 0)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_COMPLEX_ALIGN")
endif

ifeq ($(KOKKOS_INTERNAL_ENABLE_TUNING), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_TUNING")
endif

tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_LIBDL")

ifeq ($(KOKKOS_INTERNAL_USE_HWLOC), 1)
  ifneq ($(KOKKOS_CMAKE), yes)
    ifneq ($(HWLOC_PATH),)
      KOKKOS_CPPFLAGS += -I$(HWLOC_PATH)/include
      KOKKOS_LIBDIRS += -L$(HWLOC_PATH)/lib
      KOKKOS_CXXLDFLAGS += -L$(HWLOC_PATH)/lib
      KOKKOS_TPL_INCLUDE_DIRS += $(HWLOC_PATH)/include
      KOKKOS_TPL_LIBRARY_DIRS += $(HWLOC_PATH)/lib
    endif
    KOKKOS_LIBS += -lhwloc
    KOKKOS_TPL_LIBRARY_NAMES += hwloc
  endif
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_HWLOC")
endif

ifeq ($(KOKKOS_INTERNAL_ENABLE_LARGE_MEM_TESTS), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_LARGE_MEM_TESTS")
endif

tmp := $(call kokkos_append_header,"/* Optimization Settings */")

ifeq ($(KOKKOS_INTERNAL_AGGRESSIVE_VECTORIZATION), 1)
  # deprecated
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION")
endif

tmp := $(call kokkos_append_header,"/* Cuda Settings */")

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  #deprecated
  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_LDG), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_LDG_INTRINSIC")
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_LDG_INTRINSIC")
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_UVM), 1)
    # deprecated
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_UVM")
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_RELOC), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE")
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      KOKKOS_CXXFLAGS += -fcuda-rdc
      KOKKOS_LDFLAGS += -fcuda-rdc
    else
      KOKKOS_CXXFLAGS += --relocatable-device-code=true
      KOKKOS_LDFLAGS += --relocatable-device-code=true
    endif
  endif

  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    ifeq ($(shell test $(KOKKOS_INTERNAL_COMPILER_NVCC_VERSION) -ge 90; echo $$?),0)
      # This diagnostic is just plain wrong in CUDA 9
      # See https://github.com/kokkos/kokkos/issues/1470
      KOKKOS_CXXFLAGS += -Xcudafe --diag_suppress=esa_on_defaulted_function_ignored
    endif
  endif

  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_LAMBDA")
    KOKKOS_CXXFLAGS += -extended-lambda
  endif

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_LAMBDA")
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR), 1)
    ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_CONSTEXPR")
      KOKKOS_CXXFLAGS += -expt-relaxed-constexpr
    endif

    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_CUDA_CONSTEXPR")
    endif
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_DISABLE_MALLOC_ASYNC), 0)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC")
  else
    tmp := $(call kokkos_append_header,"/* $H""undef KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC */")
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1)
  ifeq ($(KOKKOS_INTERNAL_HPX_ENABLE_ASYNC_DISPATCH), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_IMPL_HPX_ASYNC_DISPATCH")
  endif
endif

# Add Architecture flags.

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV80), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARMV80")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARM_NEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    KOKKOS_CXXFLAGS += -march=armv8-a
    KOKKOS_LDFLAGS += -march=armv8-a
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV81), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARMV81")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARM_NEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    KOKKOS_CXXFLAGS += -march=armv8.1-a
    KOKKOS_LDFLAGS += -march=armv8.1-a
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_A64FX), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_A64FX")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARM_NEON")

  KOKKOS_CXXFLAGS += -march=armv8.2-a+sve
  KOKKOS_LDFLAGS += -march=armv8.2-a+sve
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    KOKKOS_CXXFLAGS += -msve-vector-bits=512
    KOKKOS_LDFLAGS += -msve-vector-bits=512
  endif
  ifeq ($(KOKKOS_INTERNAL_COMPILER_GCC), 1)
    KOKKOS_CXXFLAGS += -msve-vector-bits=512
    KOKKOS_LDFLAGS += -msve-vector-bits=512
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ZEN), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_ZEN")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -mavx2
    KOKKOS_LDFLAGS += -mavx2
  else
    KOKKOS_CXXFLAGS += -march=znver1 -mtune=znver1
    KOKKOS_LDFLAGS += -march=znver1 -mtune=znver1
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ZEN2), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_ZEN2")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -mavx2
    KOKKOS_LDFLAGS += -mavx2
  else
    KOKKOS_CXXFLAGS += -march=znver2 -mtune=znver2
    KOKKOS_LDFLAGS += -march=znver2 -mtune=znver2
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ZEN3), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_ZEN3")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -mavx2
    KOKKOS_LDFLAGS += -mavx2
  else
    KOKKOS_CXXFLAGS += -march=znver3 -mtune=znver3
    KOKKOS_LDFLAGS += -march=znver3 -mtune=znver3
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARMV80")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARMV8_THUNDERX")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARM_NEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    KOKKOS_CXXFLAGS += -march=armv8-a -mtune=thunderx
    KOKKOS_LDFLAGS += -march=armv8-a -mtune=thunderx
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX2), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARMV81")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARMV8_THUNDERX2")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ARM_NEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    KOKKOS_CXXFLAGS += -mtune=thunderx2t99 -mcpu=thunderx2t99
    KOKKOS_LDFLAGS += -mtune=thunderx2t99 -mcpu=thunderx2t99
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AVX), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -mavx
    KOKKOS_LDFLAGS  += -mavx
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  else
    # Assume that this is a really a GNU compiler.
    KOKKOS_CXXFLAGS += -mavx
    KOKKOS_LDFLAGS  += -mavx
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_POWER7), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_POWER7")

  # Assume that this is a really a GNU compiler.
  KOKKOS_CXXFLAGS += -mcpu=power7 -mtune=power7
  KOKKOS_LDFLAGS  += -mcpu=power7 -mtune=power7
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_POWER8), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_POWER8")

  KOKKOS_CXXFLAGS += -mcpu=power8 -mtune=power8
  KOKKOS_LDFLAGS  += -mcpu=power8 -mtune=power8
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_POWER9), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_POWER9")

  KOKKOS_CXXFLAGS += -mcpu=power9 -mtune=power9
  KOKKOS_LDFLAGS  += -mcpu=power9 -mtune=power9
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_HSW), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xCORE-AVX2
    KOKKOS_LDFLAGS  += -xCORE-AVX2
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  else
    # Assume that this is a really a GNU compiler.
    KOKKOS_CXXFLAGS += -march=core-avx2 -mtune=core-avx2
    KOKKOS_LDFLAGS  += -march=core-avx2 -mtune=core-avx2
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_BDW), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xCORE-AVX2
    KOKKOS_LDFLAGS  += -xCORE-AVX2
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  else
    # Assume that this is a really a GNU compiler.
    KOKKOS_CXXFLAGS += -march=core-avx2 -mtune=core-avx2 -mrtm
    KOKKOS_LDFLAGS  += -march=core-avx2 -mtune=core-avx2 -mrtm
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AVX512MIC), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX512MIC")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xMIC-AVX512
    KOKKOS_LDFLAGS  += -xMIC-AVX512
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  else
    # Asssume that this is really a GNU compiler.
    KOKKOS_CXXFLAGS += -march=knl -mtune=knl
    KOKKOS_LDFLAGS  += -march=knl -mtune=knl
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_SKL), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX512XEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xSKYLAKE
    KOKKOS_LDFLAGS  += -xSKYLAKE
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  else
    # Nothing here yet.
    KOKKOS_CXXFLAGS += -march=skylake
    KOKKOS_LDFLAGS  += -march=skylake
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_SKX), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX512XEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xCORE-AVX512
    KOKKOS_LDFLAGS  += -xCORE-AVX512
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
  else
    # Nothing here yet.
    KOKKOS_CXXFLAGS += -march=skylake-avx512 -mtune=skylake-avx512
    KOKKOS_LDFLAGS  += -march=skylake-avx512 -mtune=skylake-avx512
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ICL), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX512XEON")
  KOKKOS_CXXFLAGS += -march=icelake-client -mtune=icelake-client
  KOKKOS_LDFLAGS += -march=icelake-client -mtune=icelake-client
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ICX), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX512XEON")
  KOKKOS_CXXFLAGS += -march=icelake-server -mtune=icelake-server
  KOKKOS_LDFLAGS += -march=icelake-server -mtune=icelake-server
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_SPR), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AVX512XEON")
  KOKKOS_CXXFLAGS += -march=sapphirerapids -mtune=sapphirerapids
  KOKKOS_LDFLAGS += -march=sapphirerapids -mtune=sapphirerapids
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KNC), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KNC")
  KOKKOS_CXXFLAGS += -mmic
  KOKKOS_LDFLAGS += -mmic
endif

# Figure out the architecture flag for Cuda.
ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
   KOKKOS_INTERNAL_USE_CUDA_ARCH=1
endif
ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
   ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
     KOKKOS_INTERNAL_USE_CUDA_ARCH=1
   endif
endif
ifeq ($(KOKKOS_INTERNAL_USE_CUDA_ARCH), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG=-arch
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
		KOKKOS_INTERNAL_CUDA_ARCH_FLAG=--cuda-gpu-arch
		KOKKOS_CXXFLAGS += -x cuda
  else
    $(error Makefile.kokkos: CUDA is enabled but the compiler is neither NVCC nor Clang (got version string $(KOKKOS_CXX_VERSION)) )
  endif
  KOKKOS_INTERNAL_USE_CUDA_ARCH = 1
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY_CLANG), 1)
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG=-fopenmp
    else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG=-fopenmp --offload-arch
  endif
endif

# Do not add this flag if its the cray compiler or the nvhpc compiler.
ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY_CLANG), 0)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0)
    # Lets start with adding architecture defines
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER30), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER30")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_30
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER32), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER32")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_32
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER35), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER35")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_35
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER37), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER37")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_37
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL50")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_50
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL52")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_52
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL53")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_53
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL60), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL60")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_60
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL61), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL61")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_61
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA70), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA70")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_70
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA72), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA72")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_72
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_TURING75), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_TURING75")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_75
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE80), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE80")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_80
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE86), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE86")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_86
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ADA89), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ADA89")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_89
    endif
    ifeq ($(KOKKOS_INTERNAL_USE_ARCH_HOPPER90), 1)
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_HOPPER")
      tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_HOPPER90")
      KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_90
    endif
  endif
endif

ifneq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)

  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)
  endif
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
      KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)
    endif
  endif
endif


# Figure out the architecture flag for ROCm.
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX906), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX906")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx906
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX908), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX908")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx908
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX90A")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx90a
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX940), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX940")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx940
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX942")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx942
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX1030")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx1030
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX1100")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU")
  KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx1100
endif


ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/HIP/*.cpp)
  KOKKOS_SRC += $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_HIP.cpp
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/HIP/*.hpp)

  KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_HIP_ARCH_FLAG)
  KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_HIP_ARCH_FLAG)

  ifeq ($(KOKKOS_INTERNAL_HIP_USE_RELOC), 1)
    tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE")
    KOKKOS_CXXFLAGS+=-fgpu-rdc
    KOKKOS_LDFLAGS+=-fgpu-rdc
  else
    KOKKOS_CXXFLAGS+=-fno-gpu-rdc
    KOKKOS_LDFLAGS+=-fno-gpu-rdc
  endif
endif

# Figure out Intel architecture flags.
ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
  KOKKOS_INTERNAL_LC_BACKEND := sycl
endif
ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  KOKKOS_INTERNAL_LC_BACKEND := openmp
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GEN")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN9), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GEN9")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64_gen -X${KOKKOS_INTERNAL_LC_BACKEND}-target-backend "-device gen9"
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN11), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GEN11")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64_gen -X${KOKKOS_INTERNAL_LC_BACKEND}-target-backend "-device gen11"
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_GEN12LP), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GEN12LP")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64_gen -X${KOKKOS_INTERNAL_LC_BACKEND}-target-backend "-device gen12lp"
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_DG1), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_DG1")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64_gen -X${KOKKOS_INTERNAL_LC_BACKEND}-target-backend "-device dg1"
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_XEHP), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_XEHP")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64_gen -X${KOKKOS_INTERNAL_LC_BACKEND}-target-backend "-device 12.50.4"
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_INTEL_PVC), 1)
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_GPU")
  tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_INTEL_PVC")
  KOKKOS_INTERNAL_INTEL_ARCH_FLAG := -f${KOKKOS_INTERNAL_LC_BACKEND}-targets=spir64_gen -X${KOKKOS_INTERNAL_LC_BACKEND}-target-backend "-device 12.60.7"
endif

ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/SYCL/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/SYCL/*.hpp)

  KOKKOS_CXXFLAGS+=-fsycl -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda -fsycl-dead-args-optimization
  KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_INTEL_ARCH_FLAG)
  KOKKOS_LDFLAGS+=-fsycl
  KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_INTEL_ARCH_FLAG)
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_INTEL_ARCH_FLAG) -D__STRICT_ANSI__
  KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_INTEL_ARCH_FLAG)
endif

ifeq ($(KOKKOS_INTERNAL_DISABLE_DESUL_ATOMICS), 1)
  $(warning disable_desul_atomics option has been removed. Desul atomics cannot be disabled.)
  KOKKOS_INTERNAL_DISABLE_DESUL_ATOMICS := 0
endif
ifeq ($(KOKKOS_INTERNAL_ENABLE_DESUL_ATOMICS), 1)
  $(warning enable_desul_atomics option has been removed. Desul atomics are always enabled.)
endif
KOKKOS_CPPFLAGS+=-I$(KOKKOS_PATH)/tpls/desul/include

ifeq ($(KOKKOS_INTERNAL_DISABLE_BUNDLED_MDSPAN), 0)
  KOKKOS_CPPFLAGS+=-I$(KOKKOS_PATH)/tpls/mdspan/include
endif
tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_IMPL_MDSPAN")

KOKKOS_INTERNAL_LS_CONFIG := $(shell ls KokkosCore_config.h 2>&1)

ifeq ($(KOKKOS_INTERNAL_LS_CONFIG), KokkosCore_config.h)
  KOKKOS_INTERNAL_NEW_CONFIG := $(strip $(shell diff KokkosCore_config.h KokkosCore_config.tmp | grep -c define))
else
  KOKKOS_INTERNAL_NEW_CONFIG := 1
endif

ifneq ($(KOKKOS_INTERNAL_NEW_CONFIG), 0)
  tmp := $(shell cp KokkosCore_config.tmp KokkosCore_config.h)

  # Functions for generating config header file
  kokkos_start_config_header = $(shell sed 's~@INCLUDE_NEXT_FILE@~~g' $(KOKKOS_PATH)/cmake/KokkosCore_Config_HeaderSet.in > $1)
  kokkos_update_config_header = $(shell sed 's~@HEADER_GUARD_TAG@~$1~g' $2 > $3)
  kokkos_append_config_header = $(shell echo $1 >> $2))
  tmp := $(call kokkos_start_config_header, "KokkosCore_Config_FwdBackend.tmp")
  tmp := $(call kokkos_start_config_header, "KokkosCore_Config_SetupBackend.tmp")
  tmp := $(call kokkos_start_config_header, "KokkosCore_Config_DeclareBackend.tmp")
  tmp := $(call kokkos_start_config_header, "KokkosCore_Config_PostInclude.tmp")
  tmp := $(call kokkos_update_config_header, KOKKOS_FWD_HPP_, "KokkosCore_Config_FwdBackend.tmp", "KokkosCore_Config_FwdBackend.hpp")
  tmp := $(call kokkos_update_config_header, KOKKOS_SETUP_HPP_, "KokkosCore_Config_SetupBackend.tmp", "KokkosCore_Config_SetupBackend.hpp")
  tmp := $(call kokkos_update_config_header, KOKKOS_DECLARE_HPP_, "KokkosCore_Config_DeclareBackend.tmp", "KokkosCore_Config_DeclareBackend.hpp")
  ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_OPENMP.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_OPENMP.hpp>","KokkosCore_Config_DeclareBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_THREADS), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_THREADS.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_THREADS.hpp>","KokkosCore_Config_DeclareBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_HPX.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_HPX.hpp>","KokkosCore_Config_DeclareBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_SERIAL.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_SERIAL.hpp>","KokkosCore_Config_DeclareBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_CUDA.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_CUDA.hpp>","KokkosCore_Config_DeclareBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <setup/Kokkos_Setup_Cuda.hpp>","KokkosCore_Config_SetupBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_OPENMPTARGET.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_OPENMPTARGET.hpp>","KokkosCore_Config_DeclareBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_SYCL.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_SYCL.hpp>","KokkosCore_Config_DeclareBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <setup/Kokkos_Setup_SYCL.hpp>","KokkosCore_Config_SetupBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_HIP.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_HIP.hpp>","KokkosCore_Config_DeclareBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <setup/Kokkos_Setup_HIP.hpp>","KokkosCore_Config_SetupBackend.hpp")
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1)
    tmp := $(call kokkos_append_config_header,"$H""include <fwd/Kokkos_Fwd_OPENACC.hpp>","KokkosCore_Config_FwdBackend.hpp")
    tmp := $(call kokkos_append_config_header,"$H""include <decl/Kokkos_Declare_OPENACC.hpp>","KokkosCore_Config_DeclareBackend.hpp")
  endif
endif

KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/algorithms/src/*.hpp)

KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.cpp)
KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.cpp)

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.cpp)
  KOKKOS_SRC += $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_CUDA.cpp
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.hpp)
  ifneq ($(CUDA_PATH),)
    KOKKOS_CPPLAGS += -I$(CUDA_PATH)/include
    ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib64), 1)
      KOKKOS_LIBDIRS += -L$(CUDA_PATH)/lib64
      KOKKOS_CXXLDFLAGS += -L$(CUDA_PATH)/lib64
      KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib64
    else ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib), 1)
      KOKKOS_LIBDIRS += -L$(CUDA_PATH)/lib
      KOKKOS_CXXLDFLAGS += -L$(CUDA_PATH)/lib
      KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib
    else
      $(error Can't find CUDA library directory: no lib64 or lib directory in $(CUDA_PATH))
    endif
    KOKKOS_TPL_INCLUDE_DIRS += $(CUDA_PATH)/include
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      KOKKOS_CXXFLAGS += --cuda-path=$(CUDA_PATH)
    endif
  endif
  KOKKOS_LIBS += -lcudart -lcuda
  KOKKOS_TPL_LIBRARY_NAMES += cudart cuda
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/OpenMPTarget/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMPTarget/*.hpp)
  ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
    KOKKOS_CXXFLAGS += -Xcompiler $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG)
  else
    KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG)
  endif
  KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG)
  KOKKOS_LIBS += $(KOKKOS_INTERNAL_OPENMPTARGET_LIB)
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/OpenACC/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenACC/*.hpp)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENACC_FLAG)
  KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENACC_FLAG)
  KOKKOS_LIBS += $(KOKKOS_INTERNAL_OPENACC_LIB)
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/OpenMP/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMP/*.hpp)

  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_CXXFLAGS += -Xcompiler $(KOKKOS_INTERNAL_OPENMP_FLAG)
  else
    KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENMP_FLAG)
  endif

  KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENMP_FLAG)
  KOKKOS_LINK_FLAGS += $(KOKKOS_INTERNAL_OPENMP_FLAG)
endif

ifeq ($(KOKKOS_INTERNAL_USE_THREADS), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Threads/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Threads/*.hpp)
  KOKKOS_LIBS += -lpthread
  KOKKOS_TPL_LIBRARY_NAMES += pthread
endif

ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Serial/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Serial/*.hpp)
endif

ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/HPX/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/HPX/*.hpp)
  ifneq ($(HPX_PATH),)
    ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1)
      KOKKOS_CXXFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --cflags hpx_application_debug)
      KOKKOS_CXXLDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug)
      KOKKOS_LIBS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application_debug)
    else
      KOKKOS_CXXFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --cflags hpx_application)
      KOKKOS_CXXLDFLAGS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application)
      KOKKOS_LIBS += $(shell PKG_CONFIG_PATH=$(HPX_PATH)/lib64/pkgconfig pkg-config --libs hpx_application)
    endif
  else
    ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1)
      KOKKOS_CXXFLAGS += $(shell pkg-config --cflags hpx_application_debug)
      KOKKOS_CXXLDFLAGS += $(shell pkg-config --libs hpx_application_debug)
      KOKKOS_LIBS += $(shell pkg-config --libs hpx_application_debug)
    else
      KOKKOS_CXXFLAGS += $(shell pkg-config --cflags hpx_application)
      KOKKOS_CXXLDFLAGS += $(shell pkg-config --libs hpx_application)
      KOKKOS_LIBS += $(shell pkg-config --libs hpx_application)
    endif
  endif
  KOKKOS_TPL_LIBRARY_NAMES += hpx
endif

# With Cygwin functions such as fdopen and fileno are not defined
# when strict ansi is enabled. strict ansi gets enabled with -std=c++14
# though. So we hard undefine it here. Not sure if that has any bad side effects
# This is needed for gtest actually, not for Kokkos itself!
ifeq ($(KOKKOS_INTERNAL_OS_CYGWIN), 1)
  KOKKOS_CXXFLAGS += -U__STRICT_ANSI__
endif

# Set KokkosExtraLibs and add -lkokkos to link line
KOKKOS_EXTRA_LIBS := ${KOKKOS_LIBS}
KOKKOS_LIBS := -lkokkos ${KOKKOS_LIBS}

# Generating the <desul/atomics/Config.hpp> header
DESUL_INTERNAL_CONFIG_TMP=Desul_Config.tmp
DESUL_CONFIG_HEADER=desul/atomics/Config.hpp
desul_append_header = $(shell echo $1 >> $(DESUL_INTERNAL_CONFIG_TMP))
tmp := $(call desul_append_header, "// generated by on-demand build system by crtrott" > $(DESUL_INTERNAL_CONFIG_TMP))
tmp := $(call desul_append_header, "$H""ifndef DESUL_ATOMICS_CONFIG_HPP_")
tmp := $(call desul_append_header, "$H""define DESUL_ATOMICS_CONFIG_HPP_")
tmp := $(call desul_append_header, "")
ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_CUDA")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_CUDA */")
endif
ifeq ($(KOKKOS_INTERNAL_CUDA_USE_RELOC), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_CUDA_SEPARABLE_COMPILATION")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_CUDA_SEPARABLE_COMPILATION */")
endif

ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_HIP")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_HIP */")
endif
ifeq ($(KOKKOS_INTERNAL_HIP_USE_RELOC), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_HIP_SEPARABLE_COMPILATION")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_HIP_SEPARABLE_COMPILATION */")
endif

ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_SYCL")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_SYCL */")
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_OPENMP")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_OPENMP */")
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1)
  tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_OPENACC")
else
  tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_OPENACC */")
endif
tmp := $(call desul_append_header, "")
tmp := $(call desul_append_header, "$H""endif")

DESUL_INTERNAL_LS_CONFIG := $(shell ls $(DESUL_CONFIG_HEADER) 2>&1)

ifeq ($(DESUL_INTERNAL_LS_CONFIG), $(DESUL_CONFIG_HEADER))
  DESUL_INTERNAL_NEW_CONFIG := $(strip $(shell diff $(DESUL_CONFIG_HEADER) $(DESUL_INTERNAL_CONFIG_TMP) | grep -c define))
else
  DESUL_INTERNAL_NEW_CONFIG := 1
endif

ifneq ($(DESUL_INTERNAL_NEW_CONFIG), 0)
  tmp := $(shell mkdir -p desul/atomics)
  tmp := $(shell cp $(DESUL_INTERNAL_CONFIG_TMP) $(DESUL_CONFIG_HEADER))
endif

# Setting up dependencies.

KokkosCore_config.h:

$(DESUL_CONFIG_HEADER):

KOKKOS_CPP_DEPENDS := $(DESUL_CONFIG_HEADER) KokkosCore_config.h $(KOKKOS_HEADERS)

KOKKOS_OBJ = $(KOKKOS_SRC:.cpp=.o)
KOKKOS_OBJ_LINK = $(notdir $(KOKKOS_OBJ))

include $(KOKKOS_PATH)/Makefile.targets

kokkos-clean:
	rm -f $(KOKKOS_OBJ_LINK) $(DESUL_CONFIG_HEADER) $(DESUL_INTERNAL_CONFIG_TMP) KokkosCore_config.h KokkosCore_config.tmp libkokkos.a KokkosCore_Config_SetupBackend.hpp \
	KokkosCore_Config_FwdBackend.hpp KokkosCore_Config_DeclareBackend.hpp KokkosCore_Config_DeclareBackend.tmp \
        KokkosCore_Config_FwdBackend.tmp KokkosCore_Config_SetupBackend.tmp

libkokkos.a: $(KOKKOS_OBJ_LINK) $(KOKKOS_SRC) $(KOKKOS_HEADERS)
	ar cr libkokkos.a $(KOKKOS_OBJ_LINK)
	ranlib libkokkos.a

print-cxx-flags:
	echo "$(KOKKOS_CXXFLAGS)"

KOKKOS_LINK_DEPENDS=libkokkos.a

#we have carefully separated LDFLAGS from LIBS and LIBDIRS
#we have also separated CPPFLAGS from CXXFLAGS
#if this is not cmake, for backwards compatibility
#we just jam everything together into the CXXFLAGS and LDFLAGS
ifneq ($(KOKKOS_CMAKE), yes)
  KOKKOS_CXXFLAGS += $(KOKKOS_CPPFLAGS)
  KOKKOS_LDFLAGS += $(KOKKOS_LIBDIRS)
endif