Merge pull request #71 from jacobmerson/dev

Building on summit

Author: jacobmerson

CMakeLists.txt

@@ -39,9 +39,8 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Cray")
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL")
   # todo :  check for regular xl vs bg/q xl compiler
   #         currently assuming bg/q
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qlanglvl=extended0x -O5 -qhot=level=0 -qsimd=auto -qmaxmem=-1 -qstrict -qstrict_induction -qreport")
-  # these definitions are BGQ specific, but for now the only place we use XL compiler is on BGQ
-  #add_definitions(-DBGQ)
+  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qlanglvl=extended0x")
+  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qlanglvl=extended0x -O5 -qhot=level=0 -qsimd=auto -qmaxmem=-1 -qstrict -qstrict_induction -qreport -Wall")
 elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
   # todo :  set intel-specific cxx flags
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")

core/CMakeLists.txt

@@ -26,7 +26,7 @@ endif()
 
 add_library(las_core STATIC ${las_core_sources})
 if(${CMAKE_VERSION} VERSION_GREATER "3.8.2")
-  target_compile_features(las_core PUBLIC cxx_std_11)
+  #target_compile_features(las_core PUBLIC cxx_std_11)
 endif()
 if(MPI_COMPILE_FLAGS)
   set_target_properties(las_core PROPERTIES COMPILE_FLAGS ${MPI_CXX_COMPILE_DEFINITIONS})

scripts/config_dcs.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#mpicc=/gpfs/u/home/PASC/PASCmrsn/scratch/test_compile/mpicc
+#mpicxx=/gpfs/u/home/PASC/PASCmrsn/scratch/test_compile/mpicxx
+#mpif77=/gpfs/u/home/PASC/PASCmrsn/scratch/test_compile/mpif77
+export OMPI_CXX=xlc++_r
+
+  cmake /gpfs/u/home/PASC/PASCmrsn/barn/las/  \
+  -DCMAKE_C_COMPILER=`which mpicc` \
+  -DCMAKE_CXX_COMPILER=`which mpicxx` \
+  -DCMAKE_Fortran_COMPILER=`which mpif77` \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=0 \
+  -DSCOREC_DIR=/gpfs/u/home/PASC/PASCmrsn/scratch/dcs/install/core/lib/cmake/SCOREC/ \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX=/gpfs/u/home/PASC/PASCmrsn/scratch/dcs/install/las  \
+  -DPETSC_DIR="$PETSC_DIR" \
+  -DPETSC_ARCH="$PETSC_ARCH" \
+  -DBUILD_SPARSKIT=ON \
+  -DWITH_KOKKOS=FALSE \
+  -DBUILD_TESTS=FALSE \
+  -DCMAKE_CXX_FLAGS="-Ofast -std=c++11 -Wall" \
+  -DCMAKE_C_FLAGS="-Ofast -Wall" \
+  -DCMAKE_Fortran_FLAGS="-Ofast -Wall"
+  #-DCMAKE_C_FLAGS="-O5" \
+  #-DCMAKE_CXX_FLAGS="-O5 -sdt=c++11" \
+  #-DCMAKE_Fortran_FLAGS="-O5" \

scripts/config_erp.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+PREFIX=/gpfs/u/home/PASC/PASCmrsn/scratch/install-erp/las
+
+  cmake /gpfs/u/home/PASC/PASCmrsn/barn/las/  \
+  -DCMAKE_C_COMPILER=`which mpicc` \
+  -DCMAKE_CXX_COMPILER=`which mpicxx` \
+  -DCMAKE_Fortran_COMPILER=`which mpif77` \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=0 \
+  -DSCOREC_DIR=/gpfs/u/home/PASC/PASCmrsn/scratch/install-erp/core/lib/cmake/SCOREC/ \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+  -DPETSC_DIR="$PETSC_DIR" \
+  -DPETSC_ARCH="$PETSC_ARCH" \
+  -DBUILD_SPARSKIT=ON \
+  -DWITH_KOKKOS=FALSE \
+  -DBUILD_TESTS=FALSE

src/CMakeLists.txt

@@ -89,11 +89,11 @@ configure_file(lasConfig.h.in "${CMAKE_CURRENT_SOURCE_DIR}/lasConfig.h" @ONLY)
 set(las_headers ${las_headers} ${las_install} lasConfig.h)
 
 add_library(las STATIC ${las_sources})
-if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL")
+if("${HOST}" STREQUAL "q.ccni.rpi.edu")
   target_compile_definitions(las PUBLIC "-DBGQ")
 endif()
 if(${CMAKE_VERSION} VERSION_GREATER "3.8.2")
-  target_compile_features(las PUBLIC cxx_std_11)
+  #target_compile_features(las PUBLIC cxx_std_11)
 endif()
 if(CUDA_FOUND)
   set_target_properties(las PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
@@ -131,7 +131,7 @@ function(add_backend backend headers libraries)
     ${CMAKE_CURRENT_BINARY_DIR}/capi/las_capi_${backend}.cc
     ${headers})
   if(${CMAKE_VERSION} VERSION_GREATER "3.8.2")
-    target_compile_features(${capi_lib} PUBLIC cxx_std_11)
+    #target_compile_features(${capi_lib} PUBLIC cxx_std_11)
   endif()
   target_include_directories(${capi_lib} PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>

src/las.h

@@ -156,6 +156,14 @@ namespace las
      * @param cm The comm over which the vector is collective.
      */
     virtual Vec * create(unsigned lcl, unsigned bs, MPI_Comm cm) = 0;
+    /**
+     * Create a vector from an existing array
+     * @param data The array which the vector will use as data storage
+     * @param lcl The local number of rows (per-process in cm)
+     * @param bs The block size (should be the same over cm)
+     * @param cm The comm over which the vector is collective.
+     */
+    virtual Vec * create(scalar * data, unsigned lcl, unsigned bs, MPI_Comm cm)=0;
     virtual void destroy(Vec * v) = 0;
     /**
      * Create a vector suitable to act as the RHS vector to a
@@ -280,6 +288,30 @@ namespace las
   };
   template <class T>
   ScalarVecMult * getScalarVecMult();
+  class MatDiagonal
+  {
+    public:
+      virtual void exec(scalar s, Mat * m, Vec *& v) = 0;
+      virtual ~MatDiagonal() {}
+  };
+  template <class T>
+  MatDiagonal * getMatDiagonal();
+  class MatDiagonalInverse
+  {
+    public:
+      virtual void exec(scalar s, Mat * m, Vec *& v) = 0;
+      virtual ~MatDiagonalInverse() {}
+  };
+  template <class T>
+  MatDiagonalInverse * getMatDiagonalInverse();
+  class HadamardProduct
+  {
+    public:
+      virtual void exec(Vec * v1, Vec * v2, Vec * v3) = 0;
+      virtual ~HadamardProduct() {}
+  };
+  template <class T>
+  HadamardProduct * getHadamardProduct();
   /*
    * Finalize routines which must be called on a matrix when switching from
    * add mode to set mode

src/lasDense.cc

@@ -11,8 +11,10 @@ namespace las
       lasVec * vb = getLASVec(b);
       int nr = dns->getNumRows();
       int nc = dns->getNumCols();
+#ifndef NDEBUG
       int la = va->size();
       int lb = vb->size();
+#endif
       assert(nc == la && "Matrix columns and lhs vector length must match");
       assert(nr == lb && "Matrix rows and rhs vector lenght must match");
       for(int rw = 0; rw < nr; ++rw)
@@ -29,7 +31,9 @@ namespace las
       dnsMat * y = getDnsMat(ym);
       int x_nr = x->getNumRows();
       int x_nc = x->getNumCols();
+#ifndef NDEBUG
       int y_nr = y->getNumRows();
+#endif
       int y_nc = y->getNumCols();
       assert(x_nc == y_nr && "Matrix X cols must equal matrix Y rows");
       Sparsity * dnsty = createDensity(x_nr,y_nc);

src/lasDense_impl.h

@@ -99,6 +99,10 @@ namespace las
     {
       return createVector(lcl);
     }
+    virtual Vec * create(scalar * data, unsigned lcl, unsigned, MPI_Comm)
+    {
+      return createVector(data, lcl);
+    }
     virtual void destroy(Vec * v)
     {
       destroyVector(v);

src/lasInline.h

@@ -5,8 +5,8 @@
 #define LAS_INLINE inline
 #elif defined(__INTEL_COMPILER)
 #define LAS_INLINE __forceinline
-#elif defined(__xlc__)
-#define LAS_INLINE __attribute__((always_inline))
+#elif defined(__xlc__) || defined(__ibmxl__)
+#define LAS_INLINE inline __attribute__((always_inline))
 #elif defined(__GNUC__) || defined(__GNUG__)
 #define LAS_INLINE inline
 //__attribute__((always_inline))

src/lasPETSc_impl.h

@@ -230,6 +230,15 @@ namespace las
     CHKERRABORT(LAS_COMM_WORLD, ierr);
     return reinterpret_cast<las::Vec*>(v);
   }
+  LAS_INLINE las::Vec * createPetscVector(scalar * data, unsigned l, unsigned bs, MPI_Comm cm = LAS_COMM_WORLD)
+  {
+    ::Vec * v = new ::Vec;
+    PetscErrorCode ierr = VecCreateMPIWithArray(cm, bs, l,PETSC_DECIDE,data, v);
+    CHKERRABORT(LAS_COMM_WORLD, ierr);
+    ierr = VecSetOption(*v,VEC_IGNORE_NEGATIVE_INDICES,PETSC_TRUE);
+    CHKERRABORT(LAS_COMM_WORLD, ierr);
+    return reinterpret_cast<las::Vec*>(v);
+  }
   LAS_INLINE void destroyPetscVec(las::Vec * v)
   {
     PetscErrorCode ierr = VecDestroy(getPetscVec(v));
@@ -255,6 +264,10 @@ namespace las
     {
       return createPetscVector(lcl,bs,cm);
     }
+    virtual Vec * create(scalar * data, unsigned lcl, unsigned bs, MPI_Comm cm)
+    {
+      return createPetscVector(data, lcl,bs,cm);
+    }
     virtual Vec * createRHS(Mat * m)
     {
       return createRHSVec(m);
@@ -490,6 +503,7 @@ namespace las
       if (c == nullptr)
       {
         PetscErrorCode ierr = ::MatScale(*getPetscMat(a), s);
+        CHKERRABORT(LAS_COMM_WORLD, ierr);
       }
       else
       {