Static Libsim on Cray

This is another attempt at getting a statically linked simulation with Libsim going on Cray. I've been able to do Libsim before on a Cray but it was mainly using dynamic linking of VisIt or rolling all VisIt libraries into one to get a super-VisIt-library that was still linked dynamically. I'm also trying to get some kinks out of the system with respect to rendering.

I'm targetting cori.nersc.gov.

Compilers

The default gcc compiler is old and junky. Load some modules.

module unload PrgEnv-intel
module load PrgEnv-gnu
module load gcc
module load cmake

*NOTE - I had logic to load my preferred modules in my .cshrc.ext on edison. I thought this would apply to cori but the shell I was given on cori is bash. It took me a while to catch it.

On Cori, this will load gcc 6.2.0. On Edison, this will load gcc 6.1.0.

If you need an older gcc, do this:

module unload gcc/6.2.0
module load gcc/4.9.3

Workspace

I'll be building some of the VisIt packages manually but I may use build_visit for some others. I'll adopt the build_visit directory structures. I'll be building static, server-oriented versions of the libraries that I want to use so I'll make a thirdparty_static_server directory to contain all the 3rd party libraries.

cd
cd Development
mkdir thirdparty_static_server

If I use build_visit, I'll pass --thirdparty-path $HOME/Development/thirdparty_static_server --static in the options.

CMake

We'll need to build CMake. I ran into problems linking VisIt executables due to an -rdynamic flag. I built a CMake and edited some of its installation files to prevent adding flags we don't want during a static build.

wget https://cmake.org/files/v3.5/cmake-3.5.1.tar.gz
tar zxvf cmake-3.5.1.tar.gz
cd cmake-3.5.1
./bootstrap --prefix=$HOME/Development/thirdparty_static_server/cmake/3.5.1/linux-x86_64_gcc-6.2
make -j 4 
make -j 4 install

CMake Modifications

We're doing totally static builds but somehow, some dynamic flags are making their way into our executable link lines via the MPI link line.

  • We need to make some changes to cmake's Linux support to work around the problems.
  • Another solution could be to change VisIt's parallel executable macros so they will add any target link libraries we add before MPI libraries.

rdynamic

Edit the Linux-GNU.cmake file in the installed cmake modules:

vi $HOME/Development/thirdparty_static_server/cmake/3.5.1/linux-x86_64_gcc-6.2/share/cmake-3.5/Modules/Platform/Linux-GNU.cmake

Change this line:

set(CMAKE_SHARED_LIBRARY_LINK_${lang}_FLAGS "-rdynamic")

to:

set(CMAKE_SHARED_LIBRARY_LINK_${lang}_FLAGS "") #"-rdynamic")

Bdynamic

Edit the Linux.cmake file in the installed cmake modules:

vi $HOME/Development/thirdparty_static_server/cmake/3.5.1/linux-x86_64_gcc-6.2/share/cmake-3.5/Modules/Platform/Linux.cmake

Change this line:

set(CMAKE_${type}_LINK_DYNAMIC_C_FLAGS "-Wl,-Bdynamic")

to:

#set(CMAKE_${type}_LINK_DYNAMIC_C_FLAGS "-Wl,-Bdynamic")

OpenGL

Edit $HOME/Development/thirdparty_static_server/cmake/3.5.1/linux-x86_64_gcc-6.2/share/cmake-3.5/Modules/FindOpenGL.cmake

Change these lines, starting around line 129 from:

 if (OPENGL_gl_LIBRARY)
   if(NOT X11_FOUND)
     include(${CMAKE_CURRENT_LIST_DIR}/FindX11.cmake)
   endif()
   if (X11_FOUND)
     set (OPENGL_LIBRARIES ${X11_LIBRARIES})
   endif ()
 endif ()

to:

#  if (OPENGL_gl_LIBRARY)
#    if(NOT X11_FOUND)
#      include(${CMAKE_CURRENT_LIST_DIR}/FindX11.cmake)
#    endif()
#    if (X11_FOUND)
#      set (OPENGL_LIBRARIES ${X11_LIBRARIES})
#    endif ()
#  endif ()

This will prevent X11-related shared libraries from showing up in the link line for parallel applications. We are building our programs without X11 and with Mesa so we can link statically.

Mesa

I'm planning on taking the BG/Q route where I build a static Mesa that I will use as GL for my VTK and VisIt builds. I'm skipping the build_visit route for this since I want a little more control. Finally, why not try a more recent Mesa version. I'm using Mesa 11.1.2.

Mesa Build

I put a little script together to build Mesa:

#!/bin/sh
INSTALLDIR=$HOME/Development/thirdparty_static_server/mesa/11.1.2/linux-x86_64_gcc-6.2

tar zxvf mesa-11.1.2.tar.gz
cd mesa-11.1.2

./configure --prefix=$INSTALLDIR --disable-shared --enable-static --enable-osmesa \
--disable-dri --disable-egl --disable-glx --disable-driglx-direct --disable-gallium-llvm \
--with-gallium-drivers=swrast --disable-gles1 --disable-gles2 --disable-gbm \
--disable-vdpau --disable-xvmc --disable-driglx-direct --disable-va \
--disable-llvm-shared-libs --with-osmesa-lib-name=GL --with-sha1=libcrypto

make -j 8
make install

Mesa used the gcc 6.2 from my loaded module. Note also that the Mesa library will be called libGL.a. There also a libglapi.a that must be linked.

OpenSSL

When I tried linking an application that used the Mesa library on Cori, it failed because it could not find libcrypto. Let's build that library.

  • NOTE: My builds were able to locate a system libcrypto on edison.
wget https://www.openssl.org/source/openssl-1.1.0c.tar.gz
tar zxvf openssl-1.1.0c.tar.gz
cd openssl-1.1.0c
./config --prefix=$HOME/Development/thirdparty_static_server/openssl/1.1.0c/linux-x86_64_gcc-6.2 no-shared
make
make install

Mesa Test Application

I then tested with the osmesatest program from VisIt's diagnostics directory.

Here is the Makefile I used to build the program. Note that mesa still requires -lcrypto -ldl -lpthreads. At least it seems to be missing an X11 dependency.

MESA_DIR=$(HOME)/Development/thirdparty_static_server/mesa/11.1.2/linux-x86_64_gcc-6.2
OPENSSL_DIR=$(HOME)/Development/thirdparty_static_server/openssl/1.1.0c/linux-x86_64_gcc-6.2

MESA_CPPFLAGS=-I$(MESA_DIR)/include
MESA_LDFLAGS=-L$(MESA_DIR)/lib -L$(OPENSSL_DIR)/lib
MESA_LIBS=-lGL -lglapi -lpthread -lcrypto -ldl

CXX=CC
CXXFLAGS=-g -O2
CPPFLAGS=-I. $(MESA_CPPFLAGS) -DPARALLEL
LDFLAGS=$(MESA_LDFLAGS)
LIBS=$(MESA_LIBS)

SRC=osmesatest.cpp
OBJ=$(SRC:.cpp=.o)

all: osmesatest

osmesatest: $(OBJ)
	$(CXX) -o osmesatest $(OBJ) $(CXXFLAGS) $(LDFLAGS) $(LIBS)

clean:
	$(RM) $(OBJ) osmesatest

.cpp.o:
	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $<

Note that in the Makefile, I switched to using CC for the compiler. This is the Cray wrapper compiler. I used that since I did not want to put all the MPI-junk into the Makefile. I've been swimming upstream with MPI on Cray. Let's see if we can go through it using the wrapper compiler. This ought to use the gcc compiler since I had loaded the PrgEnv-gnu module.

The ldd output shows that CC actually made a statically-linked executable using our Mesa library.

whitlocb@cori05:~/Development/test-mesa-build/osmesatest> ldd osmesatest
	not a dynamic executable

Mesa Run

Here is the sbatch script (submit_osmesatest) I used to submit the test job.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:1:00
#SBATCH -A m636
#SBATCH -C haswell

cd $SLURM_SUBMIT_DIR

srun -n 24 ./osmesatest foo 800 800

Here is how I submitted:

sbatch submit_osmesatest

All ranks wrote out a ppm file with the right image. This is good. We've produced an X11-less Mesa that we might actually be able to use. Osmesatest.png

GLU

We haven't looked at new versions of Mesa in forever. The GLU library has been separated out into its own repo so you have to build it separately. We can piggyback off our Mesa build a little and co-install libGLU into the Mesa installation directory.

Here is a script that I used to build libGLU.

#!/bin/sh
wget ftp://ftp.freedesktop.org/pub/mesa/glu/glu-9.0.0.tar.gz
tar zxvf glu-9.0.0.tar.gz
MESA_DIR=$HOME/Development/thirdparty_static_server/mesa/11.1.2/linux-x86_64_gcc-6.2
cd glu-9.0.0
env PKG_CONFIG_LIBDIR=$MESA_DIR/lib ./configure --prefix=$MESA_DIR --disable-shared --enable-static
make -j 8 install

VTK

VTK is a central library needed for building VisIt. It does the data processing and much of the rendering and windowing. Building VTK "properly" for an HPC system can be tricky because of the lack of OpenGL, mainly. This causes problems later when you go to create a plot in the VisIt engine and it crashes while creating a vis window in case offscreen rendering needs to happen.

VTK Build

I took a similar approach to BG/Q for VTK and tell it to NOT use X11 and to use the Mesa library I built as OpenGL. This eliminated X11 dependencies for VisIt, which often prevent 100% static linking. Since I need extra control, I'll build VTK manually.

  • NOTE - When I used the Cray wrapper compilers to build VTK, I got a linker error on a little program in VTK ([ 60%] Linking C executable ../../../bin/vtkmkg3states-6.1). I switched to using gcc and VTK built and installed fine using the script below. Note how it uses our Mesa as OpenGL.
  • NOTE - I had to edit VTK-6.1.0/CMake/GenerateExportHeader.cmake and VTK-6.1.0/CMake/vtkCompilerExtras.cmake to add support for gcc 6.x.
#!/bin/bash
TPDIR=$HOME/Development/thirdparty_static_server
CMAKE=cmake
ARCH=linux-x86_64_gcc-6.2
CXX=g++
CC=gcc

MESA_DIR=$TPDIR/mesa/11.1.2/$ARCH
VTK_DIR=$TPDIR/vtk/6.1.0/$ARCH
OPENSSL_DIR=$TPDIR/openssl/1.1.0c/$ARCH

tar zxvf VTK-6.1.0.tar.gz
mkdir build_vtk
cd build_vtk

$CMAKE -DCMAKE_BUILD_TYPE:STRING=Release \
-DCMAKE_INSTALL_PREFIX:PATH=$VTK_DIR \
-DBUILD_SHARED_LIBS:BOOL=OFF \
-DCMAKE_C_COMPILER:STRING=$CC \
-DCMAKE_CXX_COMPILER:STRING=$CXX \
-DVTK_ALL_NEW_OBJECT_FACTORY:BOOL=true \
-DVTK_REPORT_OPENGL_ERRORS:BOOL=false \
-DVTK_USE_X:BOOL=OFF \
-DVTK_OPENGL_HAS_OSMESA:BOOL=ON \
-DOPENGL_INCLUDE_DIR:PATH=$MESA_DIR/include \
-DOPENGL_gl_LIBRARY:PATH="$MESA_DIR/lib/libGL.a;$MESA_DIR/lib/libglapi.a;$OPENSSL_DIR/lib/libcrypto.a;dl" \
-DOPENGL_glu_LIBRARY:PATH=$MESA_DIR/lib/libGLU.a \
-DOSMESA_LIBRARY:FILEPATH="$MESA_DIR/lib/libGL.a;$MESA_DIR/lib/libglapi.a;$OPENSSL_DIR/lib/libcrypto.a;dl" \
-DOSMESA_INCLUDE_DIR:FILEPATH=$MESA_DIR/include \
-DVTK_DEBUG_LEAKS:BOOL=false \
-DBUILD_TESTING:BOOL=false \
-DBUILD_DOCUMENTATION:BOOL=false \
-DVTK_Group_Imaging:BOOL=false \
-DVTK_Group_MPI:BOOL=false \
-DVTK_Group_Qt:BOOL=false \
-DVTK_Group_Rendering:BOOL=false \
-DVTK_Group_StandAlone:BOOL=false \
-DVTK_Group_Tk:BOOL=false \
-DVTK_Group_Views:BOOL=false \
-DModule_vtkCommonCore:BOOL=true \
-DModule_vtkFiltersFlowPaths:BOOL=true \
-DModule_vtkFiltersHybrid:BOOL=true \
-DModule_vtkFiltersModeling:BOOL=true \
-DModule_vtkGeovisCore:BOOL=true \
-DModule_vtkIOEnSight:BOOL=true \
-DModule_vtkIOGeometry:BOOL=true \
-DModule_vtkIOLegacy:BOOL=true \
-DModule_vtkIOPLY:BOOL=true \
-DModule_vtkIOXML:BOOL=true \
-DModule_vtkInteractionStyle:BOOL=true \
-DModule_vtkRenderingAnnotation:BOOL=true \
-DModule_vtkRenderingFreeType:BOOL=true \
-DModule_vtkRenderingFreeTypeOpenGL:BOOL=true \
-DModule_vtkRenderingOpenGL:BOOL=true \
-DModule_vtklibxml2:BOOL=true \
../VTK-6.1.0

make -j 8
make -j 8 install

VTK Test Application

There is a program called osmesavtktest in VisIt's diagnostics directory to help us get OSMesa working with VTK. We're going to modify it a little to not use OSMesa (and hopefully rely on our VTK's Mesa-based GL).

  1. Get the osmesavtktest program from svn.
  2. Comment out the include of vtkVisItOSMesaRenderingFactory.h
  3. Comment out the include of Utility.h
  4. Comment out the 2 lines where WaitUntilFile is called.
  5. Change the if 1 around vtkVisItOSMesaRenderingFactory::ForceMesa() to if 0 so it is not called.

The following Makefile can be used to build the program. Note that I had to experiment with the order of the VTK libraries to get a good static link on Linux and Cray. The cray wrapper compiler CC is used to get the MPI and static linking right.

TPDIR=$(HOME)/Development/thirdparty_static_server
ARCH=linux-x86_64_gcc-6.2
CXX=CC

OPENSSL_DIR=$(TPDIR)/openssl/1.1.0c/$(ARCH)

MESA_DIR=$(TPDIR)/mesa/11.1.2/$(ARCH)
MESA_CPPFLAGS=-I$(MESA_DIR)/include
MESA_LDFLAGS=-L$(MESA_DIR)/lib -L$(OPENSSL_DIR)/lib
MESA_LIBS=-lGL -lglapi -lpthread -lcrypto -ldl

VTK_DIR=$(TPDIR)/vtk/6.1.0/$(ARCH)
VTK_CPPFLAGS=-I$(VTK_DIR)/include/vtk-6.1
VTK_LDFLAGS=-L$(VTK_DIR)/lib
VTK_LIBS=-lvtkIOImage-6.1 \
-lvtkInteractionStyle-6.1 \
-lvtkRenderingOpenGL-6.1 \
-lvtkRenderingFreeTypeOpenGL-6.1 \
-lvtkRenderingFreeType-6.1 \
-lvtkRenderingCore-6.1 \
-lvtkFiltersGeometry-6.1 \
-lvtkFiltersExtraction-6.1 \
-lvtkFiltersHybrid-6.1 \
-lvtkFiltersSources-6.1 \
-lvtkCommonTransforms-6.1 \
-lvtkCommonMath-6.1 \
-lvtkCommonExecutionModel-6.1 \
-lvtkCommonDataModel-6.1 \
-lvtkCommonSystem-6.1 \
-lvtkCommonMisc-6.1 \
-lvtkCommonCore-6.1 \
-lvtkftgl-6.1 \
-lvtkfreetype-6.1 \
-lvtksys-6.1 \
-lvtkpng-6.1 \
-lvtkzlib-6.1

CXXFLAGS=-g -O2
CPPFLAGS=-I. $(VTK_CPPFLAGS) $(MESA_CPPFLAGS) -DPARALLEL
LDFLAGS=$(VTK_LDFLAGS) $(MESA_LDFLAGS)
LIBS=$(VTK_LIBS) $(MESA_LIBS)

SRC=osmesavtktest.cpp 
OBJ=$(SRC:.cpp=.o)

all: osmesavtktest

osmesavtktest: $(OBJ)
	$(CXX) -o osmesavtktest $(OBJ) $(CXXFLAGS) $(LDFLAGS) $(LIBS)

clean:
	$(RM) $(OBJ) osmesavtktest

.cpp.o:
	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $<

VTK Run

Here is the sbatch script (submit_osmesavtktest) I used to submit the test job on cori.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:1:00
#SBATCH -A m636
#SBATCH -C haswell

cd $SLURM_SUBMIT_DIR
srun -n 24 ./osmesavtktest

Here is how I submitted:

sbatch submit_osmesavtktest

All ranks ran, created their offscreen VTK window and saved a png image:

Osmesavtktest.png

We're making progress here.

Ice-T

VisIt needs Ice-T for parallel compositing. It's a little dicey setting it up sometimes. Figure this out...

  • At first I tried "env PAR_COMPILER=CC ./build_visit ..." to use the Cray wrapper compiler to locate MPI. That failed as the bv_icet.sh module would need to try running "CC --cray-print-opts=cflags" to get the options instead of -show. There's a possible enhancement for build_visit.
  • I ran "env PAR_INCLUDE=-I/opt/cray/mpt/7.3.1/gni/mpich-gnu/5.1/include ./build_visit ..." and that made build_visit happy enough about MPI.
  • build_visit creates bad ICETLibraryDepends.cmake and ICETConfig.cmake files due the invalid Mesa and MPI paths that it puts in. Since VisIt currently has its own logic to detect and use Ice-T, I'll ignore this problem.

If you want to know more about the MPI environment on the Cray, run:

CC --cray-print-opts=all

Here is the build_visit command I used to build Ice-T:

Cori

env PAR_INCLUDE=-I/opt/cray/pe/mpt/default/gni/mpich-gnu/5.1/include ./build_visit --no-visit --no-thirdparty \
    --thirdparty-path $HOME/Development/thirdparty_static_server \
    --static --server-components-only --system-cmake --icet

Edison

env PAR_INCLUDE=-I/opt/cray/mpt/default/gni/mpich-gnu/5.1/include ./build_visit --no-visit --no-thirdparty \
    --thirdparty-path $HOME/Development/thirdparty_static_server \
    --static --server-components-only --system-cmake --icet

Other VisIt Dependencies

We're doing a server-only build so we don't need Qt, Python, etc. We can still fold in some I/O libraries. Those don't usually cause many problems. Let's use build_visit for those.

Cori:

env PAR_INCLUDE=-I/opt/cray/pe/mpt/default/gni/mpich-gnu/5.1/include ./build_visit --no-visit --no-thirdparty \
   --thirdparty-path $HOME/Development/thirdparty_static_server \
   --static --server-components-only --system-cmake --icet --silo --hdf5 --szip

VisIt

Here is a config-site I created from the build_visit output and the BG/Q profiles. The intent is that we'll build a static, engine-only, VisIt build. I copied the cmake file into the src/config-site directory prior to building.

cori11.cmake

#/global/common/cori/software/cmake/3.3.2/bin/cmake
##
## ./build_visit generated host.cmake
## created: Sat Mar 19 15:05:39 PDT 2016
## system: Linux cori11 3.0.101-0.46-default #1 SMP Wed Dec 17 11:04:10 UTC 2014 (8356111) x86_64 x86_64 x86_64 GNU/Linux
## by: whitlocb

##
## Setup VISITHOME & VISITARCH variables.
##
SET(VISITHOME /global/homes/w/whitlocb/Development/thirdparty_static_server)
SET(VISITARCH linux-x86_64_gcc-6.2)

## Compiler flags.
##
VISIT_OPTION_DEFAULT(VISIT_C_COMPILER /opt/gcc/6.2.0/bin/gcc TYPE FILEPATH)
VISIT_OPTION_DEFAULT(VISIT_CXX_COMPILER /opt/gcc/6.2.0/bin/g++ TYPE FILEPATH)

##
## Static build
##
VISIT_OPTION_DEFAULT(VISIT_STATIC ON TYPE BOOL)

##
## VisIt Thread Option
##
VISIT_OPTION_DEFAULT(VISIT_THREAD OFF TYPE BOOL)

##
## MPI Settings for this machine
##
VISIT_OPTION_DEFAULT(VISIT_MPI_C_FLAGS   "-I/opt/cray/pe/mpt/default/gni/mpich-gnu/5.1/include" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_MPI_CXX_FLAGS "-I/opt/cray/pe/mpt/default/gni/mpich-gnu/5.1/include" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_MPI_LD_FLAGS "-static -static-libgcc -static-libstdc++ -pthread -Wl,-Bstatic -L/opt/cray/xc-sysroot/default/lib64" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_MPI_LIBS -Wl,--start-group /opt/cray/pe/mpt/default/gni/mpich-gnu/5.1/lib/libmpich_gnu_51.a /opt/cray/pmi/default/lib64/libpmi.a /opt/cray/alps/default/lib64/libalpsutil.a /opt/cray/alps/default/lib64/libalpslli.a /opt/cray/alps/default/lib64/libalps.a /opt/cray/wlm_detect/default/lib64/libwlm_detect.a /opt/cray/ugni/default/lib64/libugni.a /opt/cray/udreg/default/lib64/libudreg.a /opt/cray/xpmem/default/lib64/libxpmem.a pthread /usr/lib64/librt.a -Wl,--end-group)
VISIT_OPTION_DEFAULT(VISIT_PARALLEL ON TYPE BOOL)

##
## Special 
##
VISIT_OPTION_DEFAULT(VISIT_SERVER_COMPONENTS_ONLY ON TYPE BOOL)
SET(VISIT_SELECTED_DATABASE_PLUGINS "BOV;Cale;CaleHDF5;Curve2D;EnSight;Miranda;PDB;PlainText;SAMRAI;Silo;VTK")
VISIT_OPTION_DEFAULT(VISIT_USE_X            OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_USE_GLEW         OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_SLIVR            OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_PYTHON_FILTERS   OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_PYTHON_SCRIPTING OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_OPENGL_DIR       ${VISITHOME}/mesa/11.1.2/${VISITARCH})
VISIT_OPTION_DEFAULT(VISIT_CREATE_SOCKET_RELAY_EXECUTABLE ON)

##############################################################
##
## Database reader plugin support libraries
##
## The HDF4, HDF5 and NetCDF libraries must be first so that
## their libdeps are defined for any plugins that need them.
##
## For libraries with LIBDEP settings, order matters.
## Libraries with LIBDEP settings that depend on other
## Library's LIBDEP settings must come after them.
##############################################################
##

##
## VTK
##
SETUP_APP_VERSION(VTK 6.1.0)
VISIT_OPTION_DEFAULT(VISIT_VTK_DIR ${VISITHOME}/vtk/${VTK_VERSION}/${VISITARCH})
##

##
## SZIP
##
VISIT_OPTION_DEFAULT(VISIT_SZIP_DIR ${VISITHOME}/szip/2.1/${VISITARCH})

##
## HDF5
##
VISIT_OPTION_DEFAULT(VISIT_HDF5_DIR ${VISITHOME}/hdf5/1.8.14/${VISITARCH})
VISIT_OPTION_DEFAULT(VISIT_HDF5_LIBDEP ${VISITHOME}/szip/2.1/${VISITARCH}/lib sz /usr/lib z TYPE STRING)

##
## Ice-T
##
VISIT_OPTION_DEFAULT(VISIT_ICET_DIR ${VISITHOME}/icet/1.0.0/${VISITARCH})

##
## Silo
##
VISIT_OPTION_DEFAULT(VISIT_SILO_DIR ${VISITHOME}/silo/4.10.1/${VISITARCH})
VISIT_OPTION_DEFAULT(VISIT_SILO_LIBDEP HDF5_LIBRARY_DIR hdf5 ${VISIT_HDF5_LIBDEP} TYPE STRING)

VisIt Build Modifications

Edit src/CMakeLists.txt around line 1114 and change the line that defines the OpenGL library. This will be for the VISIT_STATIC case of the if-test. This change is needed so the VisIt applications that link our Mesa have their dependencies fulfilled.

SET(OPENGL_gl_LIBRARY ${VISIT_OPENGL_DIR}/lib/lib${LIBGL}.a)

We need to include the additional Mesa library dependencies from our Mesa build:

SET(OPENGL_gl_LIBRARY ${VISIT_OPENGL_DIR}/lib/lib${LIBGL}.a ${VISIT_OPENGL_DIR}/lib/libglapi.a pthread crypto dl)

Along those same lines, change

SET(OPENGL_LIBRARIES ${OPENGL_gl_LIBRARY} ${OPENGL_glu_LIBRARY})

to:

SET(OPENGL_LIBRARIES ${OPENGL_gl_LIBRARY} ${OPENGL_glu_LIBRARY})
SET(GL_LIBS ${OPENGL_gl_LIBRARY} ${OPENGL_glu_LIBRARY})

The above changes are needed to help the visitconvert_par program to link.

  • It seems wrong that the CMakeLists.txt files in the tools directory are using GL_LIBS instead of OPENGL_LIBRARIES. Then again, maybe a fix at the VTK installation level solves it all.

VisIt Build

Here is the command I'm using to configure the VisIt build:

cmake -DCMAKE_BUILD_TYPE:STRING=Release \
   -DCMAKE_INSTALL_PREFIX:PATH=$HOME/Development/SC16/install_static \
   -DVISIT_BUILD_DIAGNOSTICS:BOOL=ON ../src

mpitest

Before building VisIt, I built the mpitest diagnostic program in tools/diagnostics/mpitest to see what we get for a simple hello-world MPI program under this build. With the extra flags I added in VISIT_MPI_LD_FLAGS, I get a static application.

cd tools/diagnostics/mpitest
make
ldd ../../../exe/mpitest
	not a dynamic executable

When I run the program using this sbatch script, all ranks make the right log file.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:01:00
#SBATCH -A m636

cd $SLURM_SUBMIT_DIR

srun -n 24 /global/homes/w/whitlocb/Development/SC16/trunk/build/exe/mpitest

Log file:

% cat mpitest.0023.log
Running /global/homes/w/whitlocb/Development/SC16/trunk/build/exe/mpitest with 24 processors.
This is the log for processor 23.
Message: "HELLO FROM THE MASTER PROCESS!"

Overall Build

Once I got the issues with Mesa and VTK straightened out, I was able to issue a top-level build that succeeded in building the engine and Libsim.

% cd exe
% ls
MRUCache_test       Utility_test  exceptiontest      exprtest           networktestserver      visitconvert_par
Namescheme_test     engine_par    exceptiontest_par  mpitest            networktestserver_par  visitconvert_ser
StringHelpers_test  engine_ser    exprconfig         networktestclient  submit_batch
% ldd engine_par
	not a dynamic executable
  • There needs to be an easy, standalone way of testing the engine_par program! Right now, the easiest thing I can think of is to build an example simulation with Libsim. It would be nice if the engine itself had some builtin code to set up some pipelines and render some images without a viewer connection.

Installation

After the build completed, I issued make install. This does not install the combined Libsim runtime libraries for some reason. That has to be done manually.

cp lib/libsimV2_static_*  /global/homes/w/whitlocb/Development/SC16/install_static/2.11.0/linux-x86_64/libsim/V2/lib/

Concerns

When linking some of the applications, I'd get a message like this:

  • Does this need I'll still need to set LD_LIBRARY_PATH in my job scripts (--export=LD_LIBRARY_PATH=/opt/gcc/5.2.0/snos/lib64) ?
../../../lib/libvisitcommon.a(FileFunctions.C.o): In function `FileFunctions::ExpandPath(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)':
FileFunctions.C:(.text+0x35b6): warning: Using 'getpwnam' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
FileFunctions.C:(.text+0x3688): warning: Using 'getpwuid' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
../../../lib/libvisitcommon.a(Utility.C.o): In function `CheckHostValidity(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)':
Utility.C:(.text+0xe70): warning: Using 'gethostbyname' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
  • It seems we can ignore messages like that.

Testing Libsim

To test the Libsim build, I used the batch.c example from the VisIt Libsim examples and rendered images of a Contour plot. I used a custom Makefile using the Cray wrapper compiler to build and link the simulation.

You'll need the batch.c and extract.c and extract.h files.

Simulation Makefile

Here is the Makefile I used to build the application. Note that it uses the thirdparty_static_server installations of Mesa, Ice-T, VTK, and my installed version of VisIt for Libsim. This produces a simulation that is 100% statically linked.

  • If you were using a dynamically built Libsim, you'd only need to link in libsimV2.a to include the front end library.
CC=/opt/cray/craype/2.5.1/bin/cc
CXX=/opt/cray/craype/2.5.1/bin/CC

CFLAGS=-O3
THIRDPARTY_DIR=/global/u1/w/whitlocb/Development/thirdparty_static_server_edison
ARCH=linux-x86_64_gcc-6.1

VTK_LIBDIR=$(THIRDPARTY_DIR)/vtk/6.1.0/$(ARCH)/lib
VTK_INCDIR=$(THIRDPARTY_DIR)/vtk/6.1.0/$(ARCH)/include/vtk-6.1
VTK_LDFLAGS=-L$(VTK_LIBDIR)
VTK_LIBS=-lvtkCommonCore-6.1\
 -lvtkImagingCore-6.1\
 -lvtkRenderingCore-6.1\
 -lvtkIOLegacy-6.1\
 -lvtkIOPLY-6.1\
 -lvtkIOXML-6.1\
 -lvtkInteractionStyle-6.1\
 -lvtkRenderingOpenGL-6.1\
 -lvtkRenderingFreeTypeOpenGL-6.1\
 -lvtkFiltersHybrid-6.1\
 -lvtkFiltersModeling-6.1\
 -lvtkGeovisCore-6.1\
 -lvtkRenderingOpenGL-6.1\
 -lvtkIOGeometry-6.1\
 -lvtkjsoncpp-6.1\
 -lvtkIOXMLParser-6.1\
 -lvtkexpat-6.1\
 -lvtkInfovisLayout-6.1\
 -lvtkInfovisCore-6.1\
 -lvtkViewsCore-6.1\
 -lvtkInteractionWidgets-6.1\
 -lvtkInteractionStyle-6.1\
 -lvtkFiltersHybrid-6.1\
 -lvtkFiltersModeling-6.1\
 -lvtkImagingHybrid-6.1\
 -lvtkIOImage-6.1\
 -lvtkIOCore-6.1\
 -lvtkpng-6.1\
 -lvtktiff-6.1\
 -lvtkjpeg-6.1\
 -lvtkmetaio-6.1\
 -lvtkDICOMParser-6.1\
 -lvtkImagingGeneral-6.1\
 -lvtkImagingSources-6.1\
 -lvtkRenderingAnnotation-6.1\
 -lvtkRenderingFreeType-6.1\
 -lvtkftgl-6.1\
 -lvtkzlib-6.1\
 -lvtkImagingColor-6.1\
 -lvtkRenderingVolume-6.1\
 -lvtkRenderingCore-6.1\
 -lvtkFiltersSources-6.1\
 -lvtkFiltersGeometry-6.1\
 -lvtkFiltersExtraction-6.1\
 -lvtkFiltersGeneral-6.1\
 -lvtkCommonComputationalGeometry-6.1\
 -lvtkFiltersCore-6.1\
 -lvtkFiltersStatistics-6.1\
 -lvtkImagingFourier-6.1\
 -lvtkImagingCore-6.1\
 -lvtkCommonExecutionModel-6.1\
 -lvtkCommonDataModel-6.1\
 -lvtkCommonMisc-6.1\
 -lvtkCommonSystem-6.1\
 -lvtkCommonTransforms-6.1\
 -lvtkCommonMath-6.1\
 -lvtkCommonCore-6.1\
 -lvtkfreetype-6.1\
 -lvtksys-6.1\
 -lvtkalglib-6.1\
 -lvtkproj4-6.1

ICET_LIBDIR=$(THIRDPARTY_DIR)/icet/1.0.0/$(ARCH)/lib
ICET_INCDIR=$(THIRDPARTY_DIR)/icet/1.0.0/$(ARCH)/include
ICET_LDFLAGS=-L$(ICET_LIBDIR)
ICET_LIBS=-licet -licet_mpi -licet_strategies

LIBSIM_INCDIR=/global/homes/w/whitlocb/Development/SC16/install_static/2.11.0/linux-x86_64/libsim/V2/include
LIBSIM_LIBDIR=/global/homes/w/whitlocb/Development/SC16/install_static/2.11.0/linux-x86_64/libsim/V2/lib
LIBSIM_LDFLAGS=-L$(LIBSIM_LIBDIR)
LIBSIM_LIBS=-lsimV2_static_par

GL_LIBDIR=$(THIRDPARTY_DIR)/mesa/11.1.2/$(ARCH)/lib
GL_LDFLAGS=
GL_LIBS=$(GL_LIBDIR)/libGL.a $(GL_LIBDIR)/libglapi.a -lcrypto -ldl

CPPFLAGS=-I. -I$(LIBSIM_INCDIR) -I$(VTK_INCDIR) -DPARALLEL
LDFLAGS=$(LIBSIM_LDFLAGS) $(VTK_LDFLAGS) $(ICET_LDFLAGS) $(MPI_LDFLAGS)
LIBS=$(LIBSIM_LIBS) $(VTK_LIBS) $(ICET_LIBS) $(MPI_LIBS) $(GL_LIBS) -ldl -lpthread -lz -lm

SRC=batch.c extract.c
OBJ=$(SRC:.c=.o)

all: batch_par

batch_par: $(OBJ)
	$(CXX) -o batch_par $(OBJ) $(CFLAGS) $(LDFLAGS) $(LIBS)

clean:
	$(RM) $(OBJ) batch_par

.c.o:
	$(CC) $(CFLAGS) $(CPPFLAGS) -c $<

Simulation Run

24 cores

Here is the sbatch script I used to submit the simulation on cori. Note that the -domains I,J,K numbers used must multiply out to the number of processors used for the job submission.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:02:00
#SBATCH -A m636

cd $SLURM_SUBMIT_DIR
SIMDIR=/global/homes/w/whitlocb/Development/SC16/batch_par
srun -n 24 $SIMDIR/batch_par -domains 4,3,2 -maxcycles 10 -export 0 -render 1 -format VTK

The simulation wrote out 10 PNG files that I combined into a small animation here:

Libsim cori batch.gif

1200 cores

I ran another job at 1200 cores:

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 50
#SBATCH -t 00:10:00
#SBATCH -A m636

cd $SLURM_SUBMIT_DIR
SIMDIR=/global/homes/w/whitlocb/Development/SC16/batch_par
date
srun -n 1200 $SIMDIR/batch_par -domains 12,10,10 -maxcycles 10 -export 0 -render 1 -format VTK 
date
Sat Mar 19 19:33:10 PDT 2016
argv[0] = /global/homes/w/whitlocb/Development/SC16/batch_par/batch_par
argv[1] = -domains
argv[2] = 12,10,10
argv[3] = -maxcycles
argv[4] = 10
argv[4] = -export
argv[5] = 0
argv[6] = -render
argv[7] = 1
argv[8] = -format
argv[9] = VTK
Initialization time: 0.125405
Simulating time step: cycle=0, time=0
Saved batch0000.png
Simulating time step: cycle=1, time=0.314159
Saved batch0001.png
Simulating time step: cycle=2, time=0.628319
Saved batch0002.png
Simulating time step: cycle=3, time=0.942478
Saved batch0003.png
Simulating time step: cycle=4, time=1.25664
Saved batch0004.png
Simulating time step: cycle=5, time=1.5708
Saved batch0005.png
Simulating time step: cycle=6, time=1.88496
Saved batch0006.png
Simulating time step: cycle=7, time=2.19911
Saved batch0007.png
Simulating time step: cycle=8, time=2.51327
Saved batch0008.png
Simulating time step: cycle=9, time=2.82743
Saved batch0009.png
Sat Mar 19 19:33:38 PDT 2016

28 seconds. I like that. Way better than using dynamic libraries.

Libsim cori batch1200.gif

Lessons Learned

Here are a few things I learned.

CMake and MPI on Cray

CMake's FindMPI module doesn't like the Cray wrapper compiler for finding MPI if you happen to pass that for the CMAKE_C_COMPILER, CMAKE_CXX_COMPILER. You instead need to pass some more specific MPI variables to get CMake to include MPI in the include and link. For cori, I found it to look something like this:

#!/bin/sh

MPI_C_LIBRARIES="-Wl,--start-group /opt/cray/mpt/default/gni/mpich-gnu/5.1/lib/libmpich_gnu_51.a /opt/cray/pmi/default/lib64/libpmi.a /opt/cray/alps/default/lib64/libalpsutil.a /opt/cray/alps/default/lib64/libalpslli.a /opt/cray/alps/default/lib64/libalps.a /opt/cray/wlm_detect/default/lib64/libwlm_detect.a /opt/cray/ugni/default/lib64/libugni.a /opt/cray/udreg/default/lib64/libudreg.a /opt/cray/xpmem/default/lib64/libxpmem.a -lpthread /usr/lib64/librt.a -Wl,--end-group"

cmake -DCMAKE_C_COMPILER=/opt/gcc/5.2.0/bin/gcc \
-DCMAKE_CXX_COMPILER=/opt/gcc/5.2.0/bin/g++ \
-DMPI_C_INCLUDE_PATH=/opt/cray/mpt/7.3.1/gni/mpich-gnu/5.1/include \
-DMPI_C_LIBRARIES="$MPI_C_LIBRARIES" \
-DMPI_CXX_INCLUDE_PATH=/opt/cray/mpt/7.3.1/gni/mpich-gnu/5.1/include \
-DMPI_CXX_LIBRARIES="$MPI_C_LIBRARIES" \
/path/to/src

Detecting Libsim from CMake

Libsim doesn't currently bundle a find module for cmake. This isn't that big a deal when the normal dynamically-linked VisIt is used because you'd normally only link to libsimV2.a and libsimV2f.a, which are easy to include in a build. When VisIt is statically linked, the picture totally changes. All of a sudden, you must include the VTK, Ice-T, OpenGL, and anything else that VisIt needs to link statically. Static linking really would benefit from a FindLIBSIM.cmake module. I've taken a stab at one here so I could include a statically linked Libsim in one of my projects.

There were a couple issues:

  • VisIt's install does not install most 3rd party .a files, which makes it impossible to link to VTK in the VisIt installation.
  • Same thing with OpenGL when Mesa is used as OpenGL.

FindLIBSIM.cmake

find_package(LIBSIM REQUIRED)

add_library(libsim INTERFACE)
target_link_libraries(libsim INTERFACE ${LIBSIM_STATIC_PAR_LIBRARIES})
target_include_directories(libsim SYSTEM INTERFACE ${LIBSIM_INCLUDE_DIR})

I created a LIBSIMConfig.cmake file to be installed into the same lib directory as the Libsim .a files.

# LIBSIM_FOUND                - System has Libsim
# LIBSIM_INCLUDE_DIR          - The path to the Libsim header files
# LIBSIM_LIBRARY_DIR          - The path to the Libsim lib files
# LIBSIM_LIBRARIES            - The Libsim simV2.a file and 3rd party dependencies
# LIBSIM_FORTRAN_LIBRARIES    - The Libsim simV2.a, simV2f.a files and 3rd party dependencies
# LIBSIM_STATIC_SER_LIBRARIES - The Libsim simV2_static_ser.a file and 3rd party dependencies
# LIBSIM_STATIC_PAR_LIBRARIES - The Libsim simV2_static_par.a file and 3rd party dependencies (without MPI)

# Compute the installation prefix from this LIBSIMConfig.cmake file location.
get_filename_component(LIBSIM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH)
set(LIBSIM_MODULES_DIR "${LIBSIM_INSTALL_PREFIX}/lib")
get_filename_component(VISIT_ARCH_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
get_filename_component(VISIT_ARCH_DIR "${VISIT_ARCH_DIR}" PATH)
get_filename_component(VISIT_ARCH_DIR "${VISIT_ARCH_DIR}" PATH)
get_filename_component(VISIT_ARCH_DIR "${VISIT_ARCH_DIR}" PATH)

set(LIBSIM_INSTALL_PREFIX ${VISIT_ARCH_DIR}/libsim/V2)
set(VISIT_ARCHIVE_DIR     ${VISIT_ARCH_DIR}/archives)

SET(LIBSIM_VERSION "2.11.0")

# NOTE: The engine-only build did not install any VisIt 3rd party libraries into the
#       VisIt installation. For now, we'll reference the libraries that were used to
#       build VisIt. However, to make it easier for users to statically link Libsim,
#       we'll need to install all those 3rd party .a's.

#if static and mesa found
set(LIBSIM_GL_LIBDIR /global/homes/w/whitlocb/Development/thirdparty_static_server/mesa/11.1.2/linux-x86_64_gcc-5.2/lib)
set(LIBSIM_GL_LIBRARIES ${LIBSIM_GL_LIBDIR}/libGL.a ${LIBSIM_GL_LIBDIR}/libglapi.a crypto dl)

#if static and icet found
set(LIBSIM_ICET_LIBDIR /global/homes/w/whitlocb/Development/thirdparty_static_server/icet/1.0.0/linux-x86_64_gcc-5.2/lib)
set(LIBSIM_ICET_LIBRARIES ${LIBSIM_ICET_LIBDIR}/libicet.a ${LIBSIM_ICET_LIBDIR}/libicet_mpi.a ${LIBSIM_ICET_LIBDIR}/libicet_strategies.a)

# VTK files
set(LIBSIM_VTK_LIBDIR /global/homes/w/whitlocb/Development/thirdparty_static_server/vtk/6.1.0/linux-x86_64_gcc-5.2/lib)
set(LIBSIM_VTK_LIBRARIES
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOLegacy-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOPLY-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOXML-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInteractionStyle-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingOpenGL-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingFreeTypeOpenGL-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersHybrid-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersModeling-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkGeovisCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingOpenGL-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOGeometry-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkjsoncpp-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOXMLParser-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkexpat-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInfovisLayout-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInfovisCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkViewsCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInteractionWidgets-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInteractionStyle-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersHybrid-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersModeling-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingHybrid-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOImage-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkpng-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtktiff-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkjpeg-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkmetaio-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkDICOMParser-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingGeneral-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingSources-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingAnnotation-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingFreeType-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkftgl-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkzlib-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingColor-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingVolume-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersSources-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersGeometry-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersExtraction-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersGeneral-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonComputationalGeometry-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersStatistics-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingFourier-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonExecutionModel-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonDataModel-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonMisc-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonSystem-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonTransforms-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonMath-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkfreetype-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtksys-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkalglib-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkproj4-6.1.a
 )
 
set(LIBSIM_FOUND 1)

set(LIBSIM_INCLUDE_DIR ${LIBSIM_INSTALL_PREFIX}/include)
set(LIBSIM_LIBRARY_DIR  ${LIBSIM_INSTALL_PREFIX}/lib)

set(LIBSIM_THIRDPARTY_LIBRARIES ${LIBSIM_VTK_LIBRARIES} ${LIBSIM_ICET_LIBRARIES} ${LIBSIM_GL_LIBRARIES} z)

set(LIBSIM_LIBRARIES  ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2.a ${LIBSIM_THIRDPARTY_LIBRARIES})
set(LIBSIM_FORTRAN_LIBRARIES ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2.a ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2f.a ${LIBSIM_THIRDPARTY_LIBRARIES})
set(LIBSIM_STATIC_SER_LIBRARIES  ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2_static_ser.a ${LIBSIM_THIRDPARTY_LIBRARIES})
set(LIBSIM_STATIC_PAR_LIBRARIES  ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2_static_par.a ${LIBSIM_THIRDPARTY_LIBRARIES})

MESSAGE(STATUS "LIBSIM_FOUND = ${LIBSIM_FOUND}")
MESSAGE(STATUS "LIBSIM_VERSION = ${LIBSIM_VERSION}")
MESSAGE(STATUS "LIBSIM_INCLUDE_DIR = ${LIBSIM_INCLUDE_DIR}")
MESSAGE(STATUS "LIBSIM_LIBRARY_DIR = ${LIBSIM_LIBRARY_DIR}")
MESSAGE(STATUS "LIBSIM_LIBRARIES = ${LIBSIM_LIBRARIES}")
MESSAGE(STATUS "LIBSIM_FORTRAN_LIBRARIES = ${LIBSIM_FORTRAN_LIBRARIES}")
MESSAGE(STATUS "LIBSIM_STATIC_SER_LIBRARIES = ${LIBSIM_STATIC_SER_LIBRARIES}")
MESSAGE(STATUS "LIBSIM_STATIC_PAR_LIBRARIES = ${LIBSIM_STATIC_PAR_LIBRARIES}")

If the FindLIBSIM.cmake file is included into your CMakeLists.txt, and you define LIBSIM_DIR=/path/to/dir/containing/LIBSIMConfig.cmake on the cmake command line then cmake should locate Libsim and set up the Libsim variables. From there, you can add code like this in your CMakeLists.txt:

 target_include_directories(myapp SYSTEM PUBLIC ${LIBSIM_INCLUDE_DIR})
 target_link_libraries(myapp PRIVATE libsim)