Add HyKKT sparse KKT solver interface to HiOp

.github/workflows/spack_build.yml

@@ -4,7 +4,7 @@ on: [pull_request]
 
 env:
   SPACK_COLOR: always
-  REGISTRY: ghcr.io/llnl
+  REGISTRY: ghcr.io/ornl
   SPACK_CACHE: /opt/spack-cache
   tempdir: /opt/spack-cache
   TMP: /opt/spack-cache
@@ -163,7 +163,7 @@ jobs:
         run: spack -e . mirror set --oci-username ${{ env.USERNAME }} --oci-password "${{ secrets.GITHUB_TOKEN }}" local-buildcache
 
       - name: Trust keys
-        run: spack -e . buildcache keys --install --trust
+        run: spack -e . buildcache keys --install --trust --yes-to-all --force
 
       - name: Find external packages
         run: spack -e . external find --all --exclude python --exclude curl --exclude openssl

CMakeLists.txt

@@ -47,6 +47,7 @@ option(HIOP_USE_COINHSL "Build with sparse linear algebra" ON)
 option(HIOP_USE_GINKGO "Build with Ginkgo backend for sparse linear algebra" OFF)
 option(HIOP_USE_PARDISO "Build with PARDISO backend for sparse linear algebra" OFF)
 option(HIOP_USE_STRUMPACK "Build with STRUMPACK backend for sparse linear algebra" OFF)
+option(HIOP_USE_RESOLVE "Build with external ReSolve sparse solver support" OFF)
 option(HIOP_WITH_VALGRIND_TESTS "Run valgrind on certain integration tests" OFF)
 option(HIOP_BUILD_DOCUMENTATION "Build HiOp documentation via Doxygen" ON)
 option(HIOP_BUILD_FORTRAN_EXAMPLE "Build Fortran examples" ON)
@@ -96,9 +97,6 @@ cmake_dependent_option(
 cmake_dependent_option(
   HIOP_USE_MAGMA "Use Magma linear algebra" ON "HIOP_USE_GPU" OFF
 )
-cmake_dependent_option(
-  HIOP_USE_RESOLVE "Build with cuSolver LU support" ON "HIOP_USE_CUDA" OFF
-)
 
 add_library(hiop_tpl INTERFACE)
 add_library(hiop_options INTERFACE)
@@ -357,17 +355,6 @@ if(HIOP_SPARSE)
     endif()  
   endif(HIOP_USE_STRUMPACK)
 
-  if (HIOP_USE_RESOLVE)
-    set(HIOP_KLU_DIR CACHE PATH "Path to KLU directory")
-    include(FindKLU)
-    if(NOT KLU_LIBRARY)
-      message(STATUS "Cannot find KLU, disabling cuSOLVER LU module ...")
-      set(HIOP_USE_RESOLVE OFF CACHE BOOL "Build without cuSOLVER LU module." FORCE)
-    else()  
-      target_link_libraries(hiop_tpl INTERFACE KLU)
-    endif()  
-  endif()
-
   if(HIOP_USE_PARDISO)
     set(HIOP_PARDISO_DIR CACHE PATH "Path to PARDISO directory")
     include(FindHiopPARDISO)
@@ -394,19 +381,23 @@ if(HIOP_SPARSE)
     endif()
   endif(HIOP_USE_GINKGO)
 
-  if(NOT HIOP_USE_COINHSL AND NOT HIOP_USE_STRUMPACK AND NOT HIOP_USE_PARDISO AND NOT HIOP_USE_GINKGO)
+  if(HIOP_USE_RESOLVE)
+    include(FindReSolve)
+  endif(HIOP_USE_RESOLVE)
+
+  if(NOT HIOP_USE_COINHSL AND NOT HIOP_USE_STRUMPACK AND NOT HIOP_USE_PARDISO AND NOT HIOP_USE_GINKGO AND NOT HIOP_USE_RESOLVE)
     set(HIOP_SPARSE OFF CACHE BOOL "Build without sparse linear algebra" FORCE)
-    message(STATUS "Cannot find COINHSL, STRUMPACK, PARDISO nor GINKGO for sparse linear algebra, and the option HIOP_SPARSE will be disabled")
-endif(NOT HIOP_USE_COINHSL AND NOT HIOP_USE_STRUMPACK AND NOT HIOP_USE_PARDISO AND NOT HIOP_USE_GINKGO)
+    message(STATUS "Cannot find COINHSL, STRUMPACK, PARDISO, GINKGO, or external ReSolve support for sparse linear algebra, and the option HIOP_SPARSE will be disabled")
+  endif()
 else(HIOP_SPARSE)
   set(HIOP_USE_COINHSL OFF CACHE BOOL "Build without COINHSL" FORCE)
   set(HIOP_USE_STRUMPACK OFF CACHE BOOL "Build without STRUMPACK" FORCE)
-  set(HIOP_USE_RESOLVE OFF CACHE BOOL "Build without cuSOLVER LU module" FORCE)
+  set(HIOP_USE_RESOLVE OFF CACHE BOOL "Build without ReSolve" FORCE)
   set(HIOP_USE_PARDISO OFF CACHE BOOL "Build without PARDISO" FORCE)
   set(HIOP_USE_GINKGO OFF CACHE BOOL "Build without GINKGO" FORCE)
 endif(HIOP_SPARSE)
 
-# It simplifies linking against metis to set an internal variable
+  # It simplifies linking against metis to set an internal variable
 if(HIOP_SPARSE OR HIOP_WITH_KRON_REDUCTION)
   set(HIOP_USE_METIS ON CACHE BOOL "Link against METIS library ${METIS_LIBRARY}" FORCE)
 else()

cmake/FindReSolve.cmake

@@ -0,0 +1,82 @@
+#[[
+
+Locates an external ReSolve installation and links target `ReSolve::ReSolve`
+into `hiop_tpl`.
+
+Users may set the following variables:
+
+- ReSolve_DIR  Install prefix of ReSolve, or the directory containing
+             ReSolveConfig.cmake.
+
+Configuration stops with an error asking for ReSolve_DIR if ReSolve is not
+found or if its version is older than HIOP_RESOLVE_MIN_VERSION.
+
+]]
+
+# ReSolve_DIR is the standard CMake variable consulted by find_package in
+# CONFIG mode. Declaring it as a cache PATH makes it show up (and editable)
+# in ccmake / cmake-gui, and lets users pass -DReSolve_DIR=<path>.
+set(
+  ReSolve_DIR ""
+  CACHE PATH
+  "Directory containing ReSolveConfig.cmake (or the ReSolve install prefix)"
+)
+
+set(HIOP_RESOLVE_MIN_VERSION "0.99.2")
+
+# Remember user selected path: a failed CONFIG-mode find_package
+# overwrites ReSolve_DIR in the cache with "ReSolve_DIR-NOTFOUND".
+set(_hiop_resolve_user_dir "${ReSolve_DIR}")
+
+# Prefer a user-specified location, then fall back to the usual search paths.
+if(_hiop_resolve_user_dir)
+  find_package(ReSolve ${HIOP_RESOLVE_MIN_VERSION} CONFIG QUIET
+    HINTS ${_hiop_resolve_user_dir} ${_hiop_resolve_user_dir}/share/resolve/cmake
+    NO_DEFAULT_PATH)
+endif()
+# Only fall back to the default search paths if nothing was found in the
+# user-specified location. If an installation was found there but rejected
+# (wrong version), report that rather than silently picking another one.
+if(NOT ReSolve_FOUND AND NOT ReSolve_CONSIDERED_VERSIONS)
+  find_package(ReSolve ${HIOP_RESOLVE_MIN_VERSION} CONFIG QUIET)
+endif()
+
+if(NOT ReSolve_FOUND)
+  # Prompt the user for the location and stop so they can re-run configure.
+  if(ReSolve_CONSIDERED_VERSIONS)
+    # A ReSolve installation was found, but its version is too old.
+    list(GET ReSolve_CONSIDERED_CONFIGS 0 _hiop_resolve_cfg)
+    list(GET ReSolve_CONSIDERED_VERSIONS 0 _hiop_resolve_ver)
+    string(CONCAT _hiop_resolve_msg
+      "ReSolve version ${_hiop_resolve_ver} found at ${_hiop_resolve_cfg} "
+      "is too old; HiOp requires ReSolve >= ${HIOP_RESOLVE_MIN_VERSION}.")
+    unset(_hiop_resolve_cfg)
+    unset(_hiop_resolve_ver)
+  elseif(_hiop_resolve_user_dir)
+    set(_hiop_resolve_msg
+      "ReSolve was not found in ReSolve_DIR=\"${_hiop_resolve_user_dir}\".")
+  else()
+    set(_hiop_resolve_msg
+      "ReSolve was not found in the default search paths and ReSolve_DIR is not set.")
+  endif()
+  message(FATAL_ERROR
+    "${_hiop_resolve_msg}\n"
+    "Please enter the directory of a ReSolve >= ${HIOP_RESOLVE_MIN_VERSION} "
+    "installation in ReSolve_DIR, e.g.\n"
+    "  cmake -DReSolve_DIR=/path/to/resolve/install ${CMAKE_SOURCE_DIR}\n"
+    "or edit the ReSolve_DIR entry in ccmake/cmake-gui. ReSolve_DIR may be "
+    "the install prefix or the directory containing ReSolveConfig.cmake. "
+    "To build without ReSolve, configure with -DHIOP_USE_RESOLVE=OFF."
+  )
+elseif(NOT TARGET ReSolve::ReSolve)
+  message(FATAL_ERROR
+    "ReSolve was found at ${ReSolve_DIR}, but its configuration does not "
+    "define the expected target ReSolve::ReSolve. Check that ReSolve_DIR "
+    "points to a complete ReSolve installation."
+  )
+endif()
+
+message(STATUS "Found ReSolve ${ReSolve_VERSION}: ${ReSolve_DIR}")
+target_link_libraries(hiop_tpl INTERFACE ReSolve::ReSolve)
+unset(_hiop_resolve_user_dir)
+unset(_hiop_resolve_msg)

cmake/HiOpConfig.cmake.in

@@ -1,6 +1,14 @@
 # Author(s):
 # - Cameron Rutherford <cameron.rutherford@pnnl.gov>
 
+include(CMakeFindDependencyMacro)
+
+if(@HIOP_USE_RESOLVE@ AND NOT TARGET ReSolve::ReSolve)
+  find_dependency(ReSolve CONFIG REQUIRED
+    HINTS ${ReSolve_DIR} ENV ReSolve_DIR
+    PATH_SUFFIXES share/resolve/cmake)
+endif()
+
 include( "${CMAKE_CURRENT_LIST_DIR}/HiOpTargets.cmake" )
 
 # Configure TPLs if not already in an export set

src/Drivers/Sparse/CMakeLists.txt

@@ -18,12 +18,14 @@ add_executable(NlpSparseEx4.exe NlpSparseEx4.cpp NlpSparseEx4Driver.cpp)
 target_link_libraries(NlpSparseEx4.exe HiOp::HiOp)
 
 if(HIOP_USE_RAJA)
-  if(HIOP_USE_GPU AND HIOP_USE_CUDA AND HIOP_USE_RESOLVE)
-    set_source_files_properties(
-      NlpSparseRajaEx2.cpp 
-      NlpSparseRajaEx2Driver.cpp 
-      PROPERTIES LANGUAGE CUDA
-    )
+  if(HIOP_USE_GPU AND HIOP_USE_RESOLVE)
+    if(HIOP_USE_CUDA)
+      set_source_files_properties(
+        NlpSparseRajaEx2.cpp
+        NlpSparseRajaEx2Driver.cpp
+        PROPERTIES LANGUAGE CUDA
+      )
+    endif()
 
     add_executable(NlpSparseRajaEx2.exe  NlpSparseRajaEx2Driver.cpp  NlpSparseRajaEx2.cpp)
     target_link_libraries(NlpSparseRajaEx2.exe HiOp::HiOp)
@@ -53,9 +55,9 @@ install(
 ##########################################################
 add_test(NAME NlpSparse1_1 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx1.exe>" "500" "-selfcheck")
 add_test(NAME NlpSparse1_2 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx1.exe>" "500" "-fr" "-selfcheck")
-if(HIOP_USE_CUDA)
-  add_test(NAME NlpSparse1_3 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx1.exe>" "500" "-cusolver" "-selfcheck")
-endif(HIOP_USE_CUDA)
+if(HIOP_USE_RESOLVE)
+  add_test(NAME NlpSparse1_ReSolve COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx1.exe>" "500" "-resolve" "-selfcheck")
+endif()
 if(HIOP_USE_PARDISO)
   add_test(NAME NlpSparse1_4 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx1.exe>" "500" "-pardiso" "-selfcheck")
 endif(HIOP_USE_PARDISO)
@@ -70,9 +72,21 @@ if(HIOP_USE_GINKGO)
 endif(HIOP_USE_GINKGO)
 add_test(NAME NlpSparse2_1 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-selfcheck")
 add_test(NAME NlpSparse2_2 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-inertiafree" "-selfcheck")
-if(HIOP_USE_CUDA)
+if(HIOP_USE_RESOLVE)
+  add_test(NAME NlpSparse2_ReSolve COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-resolve" "-inertiafree" "-selfcheck")
+  add_test(NAME NlpSparse2_HyKKT COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-hykkt" "-inertiafree" "-selfcheck")
+  if(HIOP_USE_CUDA)
+    add_test(NAME NlpSparse2_ReSolve_GLU COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-resolve_cuda_glu" "-inertiafree" "-selfcheck")
+    add_test(NAME NlpSparse2_ReSolve_RF COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-resolve_cuda_rf" "-inertiafree" "-selfcheck")
+  endif()
+
+  if(HIOP_USE_HIP)
+    add_test(NAME NlpSparse2_ReSolve_HIP COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-resolve_hip" "-inertiafree" "-selfcheck")
+  endif()
+endif()
+if(HIOP_USE_CUDA AND HIOP_USE_COINHSL)
   add_test(NAME NlpSparse2_3 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-cusolver" "-inertiafree" "-selfcheck")
-endif(HIOP_USE_CUDA)
+endif()
 if(HIOP_USE_GINKGO)
   add_test(NAME NlpSparse2_4 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx2.exe>" "500" "-ginkgo" "-inertiafree" "-selfcheck")
   if(HIOP_USE_CUDA)
@@ -83,9 +97,18 @@ if(HIOP_USE_GINKGO)
   endif(HIOP_USE_HIP)
 endif(HIOP_USE_GINKGO)
 
-if(HIOP_USE_RAJA AND HIOP_USE_GPU AND HIOP_USE_CUDA AND HIOP_USE_RESOLVE)
-  add_test(NAME NlpSparseRaja2_1 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-resolve_cuda_glu")
-  add_test(NAME NlpSparseRaja2_2 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-resolve_cuda_rf")
+if(HIOP_USE_RAJA AND HIOP_USE_GPU)
+  if(HIOP_USE_CUDA)
+    if(HIOP_USE_RESOLVE)
+      add_test(NAME NlpSparseRaja2_ReSolve_GLU COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-resolve_cuda_glu")
+      add_test(NAME NlpSparseRaja2_ReSolve_RF COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-resolve_cuda_rf")
+      add_test(NAME NlpSparseRaja2_HyKKT_CUDA COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-hykkt")
+    endif()
+  endif()
+  if(HIOP_USE_HIP AND HIOP_USE_RESOLVE)
+    add_test(NAME NlpSparseRaja2_ReSolve_HIP COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-resolve_hip")
+    add_test(NAME NlpSparseRaja2_HyKKT_HIP COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseRajaEx2.exe>" "500" "-inertiafree" "-selfcheck" "-hykkt")
+  endif()
 endif()
 
 add_test(NAME NlpSparse3_1 COMMAND ${RUNCMD} "$<TARGET_FILE:NlpSparseEx3.exe>" "500" "-selfcheck")