#===============================================================================
# Copyright (C) 2020 Intel Corporation
#
# This software and the related documents are Intel copyrighted  materials,  and
# your use of  them is  governed by the  express license  under which  they were
# provided to you (License).  Unless the License provides otherwise, you may not
# use, modify, copy, publish, distribute,  disclose or transmit this software or
# the related documents without Intel's prior written permission.
#
# This software and the related documents  are provided as  is,  with no express
# or implied  warranties,  other  than those  that are  expressly stated  in the
# License.
#===============================================================================

# This file shows how to use specific cluster targets MKL::MKL_CDFT, MKL::MKL_SCALAPACK, and MKL::MKL_BLACS.
# See f_mpi/CMakeLists.txt for how to enable cluster libraries in the MKL::MKL target.

cmake_minimum_required(VERSION 3.13)
enable_testing()

# Set MKL_ROOT directory
function(define_mkl_root POTENTIAL_MKL_ROOT)
  if(EXISTS "${POTENTIAL_MKL_ROOT}/include/mkl.h")
    set(MKL_ROOT "${POTENTIAL_MKL_ROOT}" PARENT_SCOPE)
  else()
    set(MKL_ROOT "" PARENT_SCOPE)
  endif()
endfunction()

if("${MKL_ROOT}" STREQUAL "")
  if(NOT "$ENV{MKLROOT}" STREQUAL "")
    file(TO_CMAKE_PATH "$ENV{MKLROOT}" POTENTIAL_MKL_ROOT)
    define_mkl_root("${POTENTIAL_MKL_ROOT}")
  endif()
  if("${MKL_ROOT}" STREQUAL "" AND (NOT "${MKL_DIR}" STREQUAL ""))
    get_filename_component(POTENTIAL_MKL_ROOT "${MKL_DIR}/../../../" ABSOLUTE)
    define_mkl_root("${POTENTIAL_MKL_ROOT}")
  endif()
  if("${MKL_ROOT}" STREQUAL "")
    get_filename_component(MKL_CMAKE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" REALPATH)
    get_filename_component(POTENTIAL_MKL_ROOT "${MKL_CMAKE_PATH}/../../../../../" ABSOLUTE)
    define_mkl_root("${POTENTIAL_MKL_ROOT}")
  endif()
  if("${MKL_ROOT}" STREQUAL "")
    message(STATUS "Cannot infer MKL_ROOT from the MKLROOT environment variable, MKL_DIR variable, or the directory containing CMakeLists.txt.")
  endif()
endif()
if(NOT "${MKL_ROOT}" STREQUAL "")
  file(TO_CMAKE_PATH "${MKL_ROOT}" MKL_ROOT)
  message(STATUS "MKL_ROOT: ${MKL_ROOT}")
endif()

# Add cmake scripts and modules to CMake search path
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../cmake")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
if(NOT "${MKL_ROOT}" STREQUAL "")
  list(APPEND CMAKE_MODULE_PATH "${MKL_ROOT}/share/doc/mkl/examples/cmake")
endif()

# Generate domainList and ${domain}_funcList
include(generate_examples_list)

# Allow MPI scripts as compiler for examples
set(ENABLE_MPI ON)

# Define language and compiler
set(TEST_LANG C)
set(TEST_EXT c)
set(DATA_EXT dat)
include(setup_examples)

project(MKL_Examples LANGUAGES ${TEST_LANG})

find_package(MKL CONFIG REQUIRED)

# W/A for known problem in Intel(R) MPI Library 2021.1
if(MKL_MPI STREQUAL "intelmpi" AND DEFINED ENV{I_MPI_ROOT})
  if(UNIX AND $ENV{I_MPI_ROOT} MATCHES "2021.1")
    set(MPI_C_ADDITIONAL_INCLUDE_DIRS $ENV{I_MPI_ROOT}/include)
  endif()
endif()

# Force mshpc to be the first in mpi find package
if(MKL_MPI STREQUAL "msmpi" OR MKL_MPI STREQUAL "mshpc")
  set(MPI_GUESS_LIBRARY_NAME MSMPI)
endif()

find_package(MPI REQUIRED)

# Override default compile/link lines for mpi script
if(USE_MPI_SCRIPT AND WIN32)
  set(CMAKE_C_COMPILE_OBJECT "<CMAKE_C_COMPILER> <DEFINES> <INCLUDES> <FLAGS> /Fo<OBJECT> -c <SOURCE>")
  set(CMAKE_C_LINK_EXECUTABLE "<CMAKE_C_COMPILER> <OBJECTS> -o <TARGET> <LINK_LIBRARIES>")
endif()

# Try to identify MPI for correct BLACS if MKL_MPI is not defined
if(MPI_${TEST_LANG}_FOUND AND NOT MKL_MPI)
  if(UNIX AND MPI_${TEST_LANG}_COMPILER MATCHES "openmpi")
    set(CURRENT_MPI "openmpi")
  elseif(WIN32 AND MPI_${TEST_LANG}_COMPILER MATCHES "msmpi" OR MPI_${TEST_LANG}_COMPILER MATCHES "Microsoft MPI")
    set(CURRENT_MPI "msmpi")
  elseif(MPI_${TEST_LANG}_COMPILER MATCHES "mpich")
    set(CURRENT_MPI "mpich")
  elseif(MPI_${TEST_LANG}_COMPILER MATCHES ".*intel.*mpi.*")
    set(CURRENT_MPI "intelmpi")
    if(WIN32 AND MPI_LOCALONLY)
      set(MPI_LOCAL_OPT -localonly)
    endif()
  endif()
  set(MKL_MPI "${CURRENT_MPI}")
endif()

# Define target for each function from each domain
if(domainList)
foreach(domain IN LISTS domainList)
  set(TEST_INCLUDE "${MPI_${TEST_LANG}_INCLUDE_DIRS}")
  set(TEST_COPT "")
  set(TEST_LOPT "")

  list(APPEND TEST_INCLUDE "${PROJECT_SOURCE_DIR}/${domain}/sources")

  # Suppress CRT warning on Windows
  if(WIN32)
    list(APPEND TEST_COPT "-D_CRT_SECURE_NO_WARNINGS")
  endif()

  if(domain STREQUAL "cdft")
    add_library(cdft_support OBJECT ${CMAKE_CURRENT_SOURCE_DIR}/cdft/source/cdft_example_support.c)
    target_include_directories(cdft_support PUBLIC ${TEST_INCLUDE} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_compile_options(cdft_support PUBLIC ${TEST_COPT} $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    list(APPEND TEST_LOPT cdft_support)
  endif()
  if(domain STREQUAL "fftw3x_cdft")
    set(CMAKE_C_STANDARD 99)
    set(CMAKE_C_STANDARD_REQUIRED TRUE)
    set(libname ${domain}_${MKL_INTERFACE})
    set(objname ${domain}_${MKL_INTERFACE}_obj)
    file(GLOB WRAPPERS_SRC ${MKL_ROOT}/share/mkl/interfaces/${domain}/wrappers/*)
    add_library(${libname})
    add_library(${objname}_single OBJECT ${WRAPPERS_SRC})
    add_library(${objname}_double OBJECT ${WRAPPERS_SRC})
    target_compile_options(${objname}_single PRIVATE -DMKL_SINGLE
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_compile_options(${objname}_double PRIVATE -DMKL_DOUBLE
      $<TARGET_PROPERTY:MKL::MKL,INTERFACE_COMPILE_OPTIONS>)
    target_include_directories(${objname}_single PRIVATE ${TEST_INCLUDE} ${MKL_INCLUDE}/fftw
        $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_include_directories(${objname}_double PRIVATE ${TEST_INCLUDE} ${MKL_INCLUDE}/fftw
        $<TARGET_PROPERTY:MKL::MKL,INTERFACE_INCLUDE_DIRECTORIES>)
    target_link_libraries(${libname} ${objname}_single ${objname}_double)
    list(APPEND TEST_LOPT ${libname})
    list(APPEND TEST_INCLUDE ${MKL_INCLUDE}/fftw)
  endif()

  # Build target for each example
  message(STATUS "Functions list ${domain}: ${${domain}_funcList}")
  foreach(func IN LISTS ${domain}_funcList)
    set(executable "${domain}-${func}")

    file(GLOB_RECURSE ${domain}_${func}_SRC ${PROJECT_SOURCE_DIR}/${domain}/*/${func}.${TEST_EXT})
    if(NOT ${domain}_${func}_SRC)
      message(FATAL_ERROR "${domain} source file ${func}.${TEST_EXT} was not found")
    endif()

    add_executable(${executable} ${${domain}_${func}_SRC})
    target_include_directories(${executable} PUBLIC ${TEST_INCLUDE})
    target_compile_options(${executable} PUBLIC ${TEST_COPT})

    if(domain STREQUAL "cdft" OR domain STREQUAL "fftw3x_cdft")
      # MKL::MKL_CDFT = MKL_CDFT + MKL::MKL + MKL_BLACS
      target_link_libraries(${executable} PUBLIC ${TEST_LOPT} MKL::MKL_CDFT MPI::MPI_C)
    else()
      if(domain STREQUAL "scalapack" OR domain STREQUAL "pblas")
        # MKL::MKL_SCALAPACK = MKL_SCALAPACK + MKL::MKL + MKL_BLACS
        target_link_libraries(${executable} PUBLIC ${TEST_LOPT} MKL::MKL_SCALAPACK MPI::MPI_C)
      else()
        # BLACS required for all cluster functionality
        # MKL::MKL_BLACS = MKL::MKL + MKL_BLACS
        target_link_libraries(${executable} PUBLIC ${TEST_LOPT} MKL::MKL_BLACS MPI::MPI_C)
      endif()
    endif()

    # Register example as ctest
    if(domain STREQUAL "pblas" OR domain STREQUAL "scalapack")
      configure_file(${PROJECT_SOURCE_DIR}/${domain}/data/${func}.in ${CMAKE_CURRENT_BINARY_DIR} COPYONLY)
    endif()
    file(GLOB_RECURSE ${domain}_${func}_DATA ${PROJECT_SOURCE_DIR}/${domain}/*/${func}.${DATA_EXT})
    if(EXISTS ${${domain}_${func}_DATA})
      add_test(NAME ${executable} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${executable} ${${domain}_${func}_DATA} )
    else()
      add_test(NAME ${executable} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 4 ${MPI_LOCAL_OPT} ${PROJECT_BINARY_DIR}/${executable})
    endif()

    # Add Environment variables
    if(MKL_ENV)
      set_tests_properties(${executable} PROPERTIES ENVIRONMENT "${MKL_ENV}")
    endif()
  endforeach() #${domain}_funcList
endforeach() #domainList
endif() #not empty domainList
