Static Libsim on Cray / Intel

This is another iteration of creating a statically-linked Libsim on Cray. This time I'm using the Intel compiler to target KNL nodes. We're going to try and make the code issue AVX512 instructions for the KNL too. I'm documenting the process so I can make some patches for build_visit so this is ultimately seamless.

I'm targetting KNL nodes on cori.nersc.gov.

TODO List:

  • Make a patch for Mesa to deal with static linking
  • Figure out what to do with openswr build and the missing libAVX/libAVX2 libraries that seem to rely on dynamic loading. Make build_visit patch.
  • Add build_visit code to make the combined LLVM library.
  • Test all the way through on Cori.
  • Run on Titan?

Environment

The default compiler suite for my is Intel. If that's not yours then make sure you load the Intel compilers.

module load PrgEnv-intel

You'll also want to unload a couple modules to remove their libraries from the MPI settings that are gleaned from the Cray wrapper compiler (since they cause some linking issues later):

module unload darshan
module unload cray-libsci

Workspace

I'll be building some of the VisIt packages manually but I may use build_visit for some others. I'll adopt the build_visit directory structures. I'll be building static, server-oriented versions of the libraries that I want to use so I'll make a thirdparty_static_server_knl directory to contain all the 3rd party libraries.

cd
cd Development
mkdir thirdparty_static_server_knl

If I use build_visit, I'll pass --thirdparty-path $HOME/Development/thirdparty_static_server_knl --static in the options.

CMake

We'll need to build CMake. I built a CMake using build_visit and edited some of its installation files to prevent adding flags we don't want during a static build.

build_visit --no-visit --no-thirdparty --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
           --cc icc --cxx icpc --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
           --static --server-components-only --cmake --makeflags -j4

CMake Modifications

We're doing totally static builds but somehow, some dynamic flags are making their way into our executable link lines via the MPI link line.

  • We need to make some changes to cmake's Linux support to work around the problems.
  • Another solution could be to change VisIt's parallel executable macros so they will add any target link libraries we add before MPI libraries.

rdynamic

Edit the Linux-GNU.cmake file in the installed cmake modules:

vi $HOME/Development/thirdparty_static_server/cmake/3.8.1/linux-x86_64_icc-17.0.2/share/cmake-3.8/Modules/Platform/Linux-Intel.cmake

Change this line:

set(CMAKE_SHARED_LIBRARY_LINK_${lang}_FLAGS "-rdynamic")

to:

set(CMAKE_SHARED_LIBRARY_LINK_${lang}_FLAGS "") #"-rdynamic")

Bdynamic

Edit the Linux.cmake file in the installed cmake modules:

vi $HOME/Development/thirdparty_static_server/cmake/3.8.1/linux-x86_64_icc-17.0.2/share/cmake-3.8/Modules/Platform/Linux.cmake

Change this line:

set(CMAKE_${type}_LINK_DYNAMIC_C_FLAGS "-Wl,-Bdynamic")

to:

#set(CMAKE_${type}_LINK_DYNAMIC_C_FLAGS "-Wl,-Bdynamic")

LLVM

We're going to use Mesa 17 later on and that requires LLVM support. To that end, we add --llvm to the command line.

build_visit --no-visit --no-thirdparty --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
             --cc icc --cxx icpc --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
             --static --server-components-only --cmake --python --llvm --openswr --makeflags -j4

At the end, build_visit will have installed LLVM. The build creates a couple dozen .a's which all seem to have interdependencies. I found it difficult to link a Mesa test program against LLVM's many .a's so I ended up creating a single .a with the contents of all of the .a's called libLLVM.a. I basically combined all of the libLLVM*.a files and a .o file I made to satisfy some missing curses symbols.

/*icc -c -o cursesfunctions.o cursesfunctions.c */
#include <stdlib.h>
#define TERMINAL void*

int setupterm(char *term, int fildes, int *errret)
{
    return 0;
}

int tigetnum(char *capname)
{
    return 0;
}

TERMINAL *set_curterm(TERMINAL *nterm)
{
    return NULL;
}

int del_curterm(TERMINAL *oterm)
{
    return 0;
}

Here is the program to combine the libraries:

import os, sys

def combine_libraries(newlib, libs):
    alibs = [os.path.abspath(x) for x in libs ]
    os.mkdir("tmp")
    os.chdir("tmp")
    didx = 0
    oidx = 0
    for lib in alibs:
        if lib[-2:] == ".a":
            # Add the .a's .o's to the directory.
            dname = "%05d" % didx
            os.mkdir(dname)
            os.chdir(dname)
            a = os.system("ar -x %s" % lib)
            files = os.listdir(".")
            for file in files:
                if file[-2:] == ".o":
                    cmd = "mv %s ../%d.o" % (file, oidx)
                    print cmd
                    a = os.system(cmd)
                    oidx = oidx + 1
            os.chdir("..")
        elif lib[-2:] == ".o":
            # Add the .o to the directory.
            cmd = "cp %s ." % lib
            print cmd
            a = os.system(cmd)
            oidx = oidx + 1
        didx = didx + 1
    os.chdir("..")
    cmd = "ar cr %s tmp/*.o" % newlib
    a = os.system(cmd)
    cmd = "ranlib %s" % newlib
    a = os.system(cmd)
    cd = "rm -rf tmp"
    a = os.system(cmd)

combine_libraries(sys.argv[1], sys.argv[2:])

I cd'd into the LLVM installation's lib directory and did this:

icc -c -o cursesfunctions.o cursesfunctions.c
python combine_libraries.py libLLVM.a libLLVM*.a cursesfunctions.o

This produced the libLLVM.a that I wanted.

Mesa

I'm using recently added support for OpenSWR in build_visit which can now build Mesa 17. I'll build a static Mesa that I will use as GL for my VTK and VisIt builds.

Mesa Build

We can use build_visit to build our static Mesa. This produced a Mesa library called libOSMesa.a that contained the Mesa API. We'll try using that as the GL library for VTK and VisIt.

build_visit --no-visit --no-thirdparty --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
            --cc icc --cxx icpc --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
            --static --server-components-only --cmake --python --llvm --openswr --makeflags -j4

I did run into a problem where Mesa could not build fully. To work around this for the time being, I did comment out 2 CXXLD commands in the Makefiles. These failed libraries later cause a runtime problem if openswr is used.

  CXXLD    libswrAVX.la
libtool: link: can not build a shared library
libtool: link: See the libtool documentation for more information.
libtool: link: Fatal configuration error.
Makefile:1243: recipe for target 'libswrAVX.la' failed
make[5]: *** [libswrAVX.la] Error 1

Mesa Test Application

I then tested with the osmesatest program from VisIt's diagnostics directory.

Here is the Makefile I used to build the program.

MESA_DIR=$(HOME)/Development/thirdparty_static_server_cori_knl/openswr/17.0.6/linux-x86_64_icc-17.0.2
LLVM_DIR=$(HOME)/Development/thirdparty_static_server_cori_knl/llvm/4.0.0/linux-x86_64_icc-17.0.2

MESA_CPPFLAGS=-I$(MESA_DIR)/include
MESA_LDFLAGS=-L$(MESA_DIR)/lib -L$(LLVM_DIR)/lib
MESA_LIBS=-lOSMesa -lLLVM 

CXX=CC
CXXFLAGS=-g -O2
CPPFLAGS=-I. $(MESA_CPPFLAGS) -DPARALLEL
LDFLAGS=$(MESA_LDFLAGS)
LIBS=$(MESA_LIBS)

SRC=osmesatest.cpp
OBJ=$(SRC:.cpp=.o)

all: osmesatest

osmesatest: $(OBJ)
	$(CXX) -o osmesatest $(OBJ) $(CXXFLAGS) $(LDFLAGS) $(LIBS)

clean:
	$(RM) $(OBJ) osmesatest

.cpp.o:
	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $<

Note that in the Makefile, I switched to using CC for the compiler. This is the Cray wrapper compiler. I used that since I did not want to put all the MPI-junk into the Makefile. I've been swimming upstream with MPI on Cray.

The ldd output shows that CC actually made a statically-linked executable using our Mesa library.

whitlocb@cori05:~/Development/test-mesa-build/osmesatest> ldd osmesatest
	not a dynamic executable

Mesa Run

Here is the sbatch script (submit_osmesatest) I used to submit the test job.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:1:00
#SBATCH -A m636
#SBATCH -C knl

cd $SLURM_SUBMIT_DIR
time
srun -n 24 ./osmesatest foo 800 800
time

Here is how I submitted:

sbatch submit_osmesatest

All ranks wrote out a ppm file with the right image.
Osmesatest.png

OpenSWR

The above Mesa test program run used a regular driver in Mesa instead of the OpenSWR driver. OpenSWR must be activated by an environment variable. Here is another sbatch script where I tried to activate OpenSWR.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:10:00
#SBATCH -A m636
#SBATCH -C knl

cd $SLURM_SUBMIT_DIR
export GALLIUM_DRIVER=swr
time
srun -n 24 ./osmesatest foo 800 800
time

Unfortunately, Mesa isn't quite right for OpenSWR in a static build. I get errors from the job failing to load a shared library, whose contents ought to be compiled into Mesa but aren't. The Mesa installation does not include a libswrAVX2.so when built statically. (I disabled building these files to get Mesa to build statically!)

user	0m0.01s
sys	0m0.02s
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR detected AVX2
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
argv[0] = $HOME/Development/MAIN/cori_build_static_icc/testmesa/try2/./osmesatest
argv[1] = foo
argv[2] = 800
argv[3] = 800
SWR library load failure: libswrAVX2.so: cannot open shared object file: No such file or directory
srun: error: nid03387: tasks 0-23: Exited with exit code 255
srun: Terminating job step 6185324.0
user	0m0.31s
sys	0m0.08s
whitlocb@cori03:


GLU

I'm using Mesa 17 for OpenSWR support. A modern version of Mesa like this does not include the GLU library. The GLU library has been separated out into its own repo so you have to build it separately. We can piggyback off our Mesa build a little and co-install libGLU into the Mesa installation directory.

I added GLU support to build_visit.

build_visit --no-visit --no-thirdparty --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
            --cc icc --cxx icpc --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
            --static --server-components-only --cmake --python --llvm --openswr --glu \
            --makeflags -j4

VTK

VTK is a central library needed for building VisIt. It does the data processing and much of the rendering and windowing. Building VTK "properly" for an HPC system can be tricky because of the lack of OpenGL, mainly. This causes problems later when you go to create a plot in the VisIt engine and it crashes while creating a vis window in case offscreen rendering needs to happen.

VTK Build

I used our Mesa build as GL for VTK. I put these modifications into build_visit so they execute for static builds where we are doing server-only or engine-only builds and we've requested mesa or openswr.

build_visit --no-visit --no-thirdparty --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
           --cc icc --cxx icpc --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
           --static --server-components-only --cmake --python --llvm --openswr --glu --vtk \
           --makeflags -j4

VTK Test Application

There is a program called osmesavtktest in VisIt's diagnostics directory to help us get OSMesa working with VTK. We're going to modify it a little to not use OSMesa (and hopefully rely on our VTK's Mesa-based GL).

  1. Get the osmesavtktest program from svn.
  2. Comment out the include of vtkVisItOSMesaRenderingFactory.h
  3. Comment out the include of Utility.h
  4. Comment out the 2 lines where WaitUntilFile is called.
  5. Change the if 1 around vtkVisItOSMesaRenderingFactory::ForceMesa() to if 0 so it is not called.

The following Makefile can be used to build the program. Note that I had to experiment with the order of the VTK libraries to get a good static link on Linux and Cray. The cray wrapper compiler CC is used to get the MPI and static linking right.

TPDIR=$(HOME)/Development/thirdparty_static_server_knl
ARCH=linux-x86_64_icc-17.0.2
CXX=CC

LLVM_DIR=$(TPDIR)/llvm/4.0.0/$(ARCH)
LLVM_CPPFLAGS=-I$(LLVM_DIR)/include
LLVM_LDFLAGS=-L$(LLVM_DIR)/lib
LLVM_LIBS=-lLLVM

MESA_DIR=$(TPDIR)/openswr/17.0.6/$(ARCH)
MESA_CPPFLAGS=-I$(MESA_DIR)/include
MESA_LDFLAGS=-L$(MESA_DIR)/lib
MESA_LIBS=-lOSMesa 

VTK_DIR=$(TPDIR)/vtk/6.1.0/$(ARCH)
VTK_CPPFLAGS=-I$(VTK_DIR)/include/vtk-6.1
VTK_LDFLAGS=-L$(VTK_DIR)/lib
VTK_LIBS=-lvtkIOImage-6.1 \
-lvtkInteractionStyle-6.1 \
-lvtkRenderingOpenGL-6.1 \
-lvtkRenderingFreeTypeOpenGL-6.1 \
-lvtkRenderingFreeType-6.1 \
-lvtkRenderingCore-6.1 \
-lvtkFiltersGeometry-6.1 \
-lvtkFiltersExtraction-6.1 \
-lvtkFiltersHybrid-6.1 \
-lvtkFiltersSources-6.1 \
-lvtkCommonTransforms-6.1 \
-lvtkCommonMath-6.1 \
-lvtkCommonExecutionModel-6.1 \
-lvtkCommonDataModel-6.1 \
-lvtkCommonSystem-6.1 \
-lvtkCommonMisc-6.1 \
-lvtkCommonCore-6.1 \
-lvtkftgl-6.1 \
-lvtkfreetype-6.1 \
-lvtksys-6.1 \
-lvtkpng-6.1 \
-lvtkzlib-6.1

CXXFLAGS=-g -O2
CPPFLAGS=-I. $(VTK_CPPFLAGS) $(MESA_CPPFLAGS) $(LLVM_CPPFLAGS) -DPARALLEL
LDFLAGS=$(VTK_LDFLAGS) $(MESA_LDFLAGS) $(LLVM_LDFLAGS)
LIBS=$(VTK_LIBS) $(MESA_LIBS) $(LLVM_LIBS)

SRC=osmesavtktest.cpp 
OBJ=$(SRC:.cpp=.o)

all: osmesavtktest

osmesavtktest: $(OBJ)
	$(CXX) -o osmesavtktest $(OBJ) $(CXXFLAGS) $(LDFLAGS) $(LIBS)

clean:
	$(RM) $(OBJ) osmesavtktest

.cpp.o:
	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $<

VTK Run

Here is the sbatch script (submit_osmesavtktest) I used to submit the test job on cori.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:10:00
#SBATCH -A m636
#SBATCH -C knl
 
cd $SLURM_SUBMIT_DIR
srun -n 68 ./osmesavtktest

Here is how I submitted:

sbatch submit_osmesavtktest

All ranks ran, created their offscreen VTK window and saved a png image:

Osmesavtktest.png

We're making progress here.

Ice-T

VisIt needs Ice-T for parallel compositing. It's a little dicey setting it up sometimes.

If you want to know more about the MPI environment on the Cray, run:

CC --cray-print-opts=all

The Cray programming environment includes some MPI environment variables. We can use one to prime the PAR_INCLUDE variable that build_visit wants.

env PAR_INCLUDE=-I$CRAY_MPICH_DIR/include ./build_visit --no-visit --no-thirdparty \
    --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
    --cc icc --cxx icpc --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
    --static --server-components-only --cmake --python --llvm --openswr --glu --vtk --icet --parallel \
    --makeflags -j4

Other VisIt Dependencies

We're doing a server-only build so we don't need Qt, Python, etc. We can still fold in some I/O libraries. Those don't usually cause many problems. Let's use build_visit for those.

Cori:

env PAR_INCLUDE=-I$CRAY_MPICH_DIR/include ./build_visit --no-visit --no-thirdparty \
    --thirdparty-path $HOME/Development/thirdparty_static_server_knl \
    --cc icc --cxx icpc --fc ifort --cflags "-xCORE-AVX2 -axMIC-AVX512" --cxxflags "-xCORE-AVX2 -axMIC-AVX512" \
    --static --server-components-only --cmake --python --llvm --openswr --glu --vtk --icet --parallel \
    --silo --hdf5 --szip \
    --makeflags -j4
  • NOTE: I often need the HDF5 fortran module to link simulations I use so I added --fc ifort to the arguments. I extended build_visit to support --fc to set the Fortran compiler and enable building Fortran.

VisIt

Here is a config-site generated by a build_visit that I've been modifying to work better on Cray with static linking. The intent is that we'll build a static, engine-only, VisIt build. I copied the cmake file into the src/config-site directory prior to building.

#/global/homes/w/whitlocb/Development/thirdparty_static_server_cori_knl/cmake/3.8.1/linux-x86_64_icc-17.0.2/bin/cmake
##
## ./build_visit generated host.cmake
## created: Mon Aug  7 13:51:44 PDT 2017
## system: Linux cori09 3.12.61-52.77.1.13154.1.PTF.1043232-default #1 SMP Fri Jun 23 07:40:38 UTC 2017 (aa51063) x86_64 x86_64 x86_64 GNU/Linux
## by: whitlocb

##
## Setup VISITHOME & VISITARCH variables.
##
SET(VISITHOME /global/homes/w/whitlocb/Development/thirdparty_static_server_cori_knl)
SET(VISITARCH linux-x86_64_icc-17.0.2)

## Compiler flags.
##
VISIT_OPTION_DEFAULT(VISIT_C_COMPILER icc TYPE FILEPATH)
VISIT_OPTION_DEFAULT(VISIT_CXX_COMPILER icpc TYPE FILEPATH)
VISIT_OPTION_DEFAULT(VISIT_FORTRAN_COMPILER ifort TYPE FILEPATH)
VISIT_OPTION_DEFAULT(VISIT_C_FLAGS "-m64  -xCORE-AVX2 -axMIC-AVX512" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_CXX_FLAGS "-m64  -xCORE-AVX2 -axMIC-AVX512" TYPE STRING)

##
## Parallel Build Setup.
##
VISIT_OPTION_DEFAULT(VISIT_PARALLEL ON TYPE BOOL)
## (configured w/ user provided CXX (PAR_INCLUDE) & LDFLAGS (PAR_LIBS) flags)
VISIT_OPTION_DEFAULT(VISIT_MPI_C_FLAGS   " -I/opt/cray/pe/mpt/7.4.4/gni/sma/include -I/opt/cray/pe/mpt/7.4.4/gni/mpich-intel/16.0/include -I/opt/cray/rca/2.1.6_g2c60fbf-2.265/include -I/opt/cray/alps/6.3.4-2.21/include -I/opt/cray/xpmem/2.1.1_gf9c9084-2.38/include -I/opt/cray/gni-headers/5.0.11-2.2/include -I/opt/cray/dmapp/7.1.1-39.37/include -I/opt/cray/pe/pmi/5.0.10-1.0000.11050.0.0.ari/include -I/opt/cray/ugni/6.0.15-2.2/include -I/opt/cray/udreg/2.3.2-7.54/include -I/opt/cray/wlm_detect/1.2.1-3.10/include -I/opt/cray/krca/2.1.3_g923f58b-2.244/include -I/opt/cray-hss-devel/8.0.0/include" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_MPI_CXX_FLAGS " -I/opt/cray/pe/mpt/7.4.4/gni/sma/include -I/opt/cray/pe/mpt/7.4.4/gni/mpich-intel/16.0/include -I/opt/cray/rca/2.1.6_g2c60fbf-2.265/include -I/opt/cray/alps/6.3.4-2.21/include -I/opt/cray/xpmem/2.1.1_gf9c9084-2.38/include -I/opt/cray/gni-headers/5.0.11-2.2/include -I/opt/cray/dmapp/7.1.1-39.37/include -I/opt/cray/pe/pmi/5.0.10-1.0000.11050.0.0.ari/include -I/opt/cray/ugni/6.0.15-2.2/include -I/opt/cray/udreg/2.3.2-7.54/include -I/opt/cray/wlm_detect/1.2.1-3.10/include -I/opt/cray/krca/2.1.3_g923f58b-2.244/include -I/opt/cray-hss-devel/8.0.0/include" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_MPI_LD_FLAGS  " -L/opt/cray/pe/mpt/7.4.4/gni/sma/lib64 -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.4.4/gni/mpich-intel/16.0/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.4.4/gni/mpich-intel/16.0/lib -L/opt/cray/rca/2.1.6_g2c60fbf-2.265/lib64 -L/opt/cray/alps/6.3.4-2.21/lib64 -L/opt/cray/xpmem/2.1.1_gf9c9084-2.38/lib64 -L/opt/cray/dmapp/7.1.1-39.37/lib64 -L/opt/cray/pe/pmi/5.0.10-1.0000.11050.0.0.ari/lib64 -L/opt/cray/ugni/6.0.15-2.2/lib64 -L/opt/cray/udreg/2.3.2-7.54/lib64 -L/opt/cray/pe/atp/2.1.0/libApp -L/opt/cray/wlm_detect/1.2.1-3.10/lib64 -Wl,--no-as-needed,-lAtpSigHandler,-lAtpSigHCommData -Wl,--undefined=_ATP_Data_Globals -Wl,--undefined=__atpHandlerInstall" TYPE STRING)
VISIT_OPTION_DEFAULT(VISIT_MPI_LIBS         pthread mpichcxx_intel rt ugni pthread pmi m dl mpich_intel rt ugni pthread pmi m dl sma pmi dmapp pthread xpmem udreg pmi pthread alpslli pthread wlm_detect ugni pthread alpsutil pthread rca TYPE STRING)

##
## Static build
##
VISIT_OPTION_DEFAULT(VISIT_STATIC ON TYPE BOOL)
# Force static executables on Cray to be 100% statically linked.
SET(VISIT_EXE_LINKER_FLAGS "-static -static-libgcc -static-libstdc++ -pthread -Wl,-Bstatic")

##
## Server components only
##
VISIT_OPTION_DEFAULT(VISIT_SERVER_COMPONENTS_ONLY ON TYPE BOOL)
# Set up VisIt to use OpenSWR as GL.
VISIT_OPTION_DEFAULT(VISIT_USE_X    OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_USE_GLEW OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_SLIVR    OFF TYPE BOOL)
VISIT_OPTION_DEFAULT(VISIT_OPENGL_DIR  /global/homes/w/whitlocb/Development/thirdparty_static_server_cori_knl/openswr/17.0.6/linux-x86_64_icc-17.0.2)
VISIT_OPTION_DEFAULT(VISIT_OPENGL_LIBRARY /global/homes/w/whitlocb/Development/thirdparty_static_server_cori_knl/openswr/17.0.6/linux-x86_64_icc-17.0.2/lib/libOSMesa.a;/global/homes/w/whitlocb/Development/thirdparty_static_server_cori_knl/llvm/4.0.0/linux-x86_64_icc-17.0.2/lib/libLLVM.a)
VISIT_OPTION_DEFAULT(VISIT_GLU_LIBRARY /global/homes/w/whitlocb/Development/thirdparty_static_server_cori_knl/openswr/17.0.6/linux-x86_64_icc-17.0.2/lib/libGLU.a)
VISIT_OPTION_DEFAULT(VISIT_CREATE_SOCKET_RELAY_EXECUTABLE ON)

##
## VisIt Thread Option
##
VISIT_OPTION_DEFAULT(VISIT_THREAD OFF TYPE BOOL)

##############################################################
##
## Database reader plugin support libraries
##
## The HDF4, HDF5 and NetCDF libraries must be first so that
## their libdeps are defined for any plugins that need them.
##
## For libraries with LIBDEP settings, order matters.
## Libraries with LIBDEP settings that depend on other
## Library's LIBDEP settings must come after them.
##############################################################
##

##
## Python
##
VISIT_OPTION_DEFAULT(VISIT_PYTHON_DIR ${VISITHOME}/python/2.7.11/${VISITARCH})

##
## VTK
##
SETUP_APP_VERSION(VTK 6.1.0)
VISIT_OPTION_DEFAULT(VISIT_VTK_DIR ${VISITHOME}/vtk/${VTK_VERSION}/${VISITARCH})

##
## SZIP
##
VISIT_OPTION_DEFAULT(VISIT_SZIP_DIR ${VISITHOME}/szip/2.1/${VISITARCH})

##
## HDF5
##
VISIT_OPTION_DEFAULT(VISIT_HDF5_DIR ${VISITHOME}/hdf5/1.8.14/${VISITARCH})
VISIT_OPTION_DEFAULT(VISIT_HDF5_LIBDEP ${VISITHOME}/szip/2.1/${VISITARCH}/lib sz /usr/lib z TYPE STRING)

##
## Ice-T
##
VISIT_OPTION_DEFAULT(VISIT_ICET_DIR ${VISITHOME}/icet/1.0.0/${VISITARCH})

##
## LLVM
##
VISIT_OPTION_DEFAULT(VISIT_LLVM_DIR ${VISITHOME}/llvm/4.0.0/${VISITARCH})
##

##
## Silo
##
VISIT_OPTION_DEFAULT(VISIT_SILO_DIR ${VISITHOME}/silo/4.10.1/${VISITARCH})
VISIT_OPTION_DEFAULT(VISIT_SILO_LIBDEP HDF5_LIBRARY_DIR hdf5 ${VISIT_HDF5_LIBDEP} /usr/lib z TYPE STRING)

VisIt Build Modifications

I made some changes to VisIt's build to work better on Cray.

  • I modified how VisIt detects OpenGL so we can pass actual paths to the OpenGL and GLU libraries so we can store that in a config-site file instead of having to modify the build. The variables are: VISIT_OPENGL_LIBRARY and VISIT_GLU_LIBRARY. In this build, I built with OpenSWR (Mesa 17 with OpenSWR support) and it requires linking with LLVM. I pass both libOSMesa.a and LLVM.a as a semi-colon separated list of libraries for my OpenGL library.
  • I noticed that some serial applications were not being linked statically (system libs, etc) so I made some changes to the build so programs like the mdserver and vcl use VISIT_EXE_LINKER_FLAGS which we can set in the config-site. I use the variable to pass some additional flags that tell the compiler to link C++ runtimes, etc statically. Note that I only added VISIT_EXE_LINKER_FLAGS to some of the serial executables in the build since I was doing a server-only build and thus did not care about the gui or viewer.
  • For XDB support, I set VISIT_ENABLE_XDB in the config-site file and then made a few more changes:
    • I had to decouple XDB from VISIT_STATIC in the databases/CMakeLists.txt
    • I changed databases/FieldViewXDB/CMakeLists.txt so it sets XDB_PLATFORM to Linux64/static/icc.
    • I copied a build of XDB for icc into Linux64/static/icc. Copy ~/xdb_static_libraries.tar to databases/FieldViewXDB/lib/Linux64 and unzip it.

VisIt Build

Here is the command I'm using to configure the VisIt build:

cmake -DCMAKE_BUILD_TYPE:STRING=Release \
   -DCMAKE_INSTALL_PREFIX:PATH=$HOME/Development/install_cori_knl \
   ../src


Installation

After the build completed, I issued make install. This does not install the combined Libsim runtime libraries for some reason. That has to be done manually.

cp lib/libsimV2_static_*  /global/homes/w/whitlocb/Development/install_cori_knl/2.13.0/linux-x86_64/libsim/V2/lib/

I copied my NERSC customlauncher script into the installation's 2.13.0/bin directory so I could try client/server to the engine running on the KNL nodes.

Testing Libsim

To test the Libsim build, I used the batch.c example from the VisIt Libsim examples and rendered images of a Contour plot. I used a custom Makefile using the Cray wrapper compiler to build and link the simulation.

You'll need the batch.c and extract.c and extract.h files.

Simulation Makefile

Here is the Makefile I used to build the application. Note that it uses the thirdparty_static_server_knl installations of Mesa, LLVM, Ice-T, VTK, and my installed version of VisIt for Libsim. This produces a simulation that is 100% statically linked.

  • If you were using a dynamically built Libsim, you'd only need to link in libsimV2.a to include the front end library.
# As of August 7, 2017
CC=/opt/cray/pe/craype/2.5.7/bin/cc
CXX=/opt/cray/pe/craype/2.5.7/bin/CC

CFLAGS=-O3
THIRDPARTY_DIR=$(HOME)/Development/thirdparty_static_server_knl
ARCH=linux-x86_64_icc-17.0.2

VTK_LIBDIR=$(THIRDPARTY_DIR)/vtk/6.1.0/$(ARCH)/lib
VTK_INCDIR=$(THIRDPARTY_DIR)/vtk/6.1.0/$(ARCH)/include/vtk-6.1
VTK_LDFLAGS=-L$(VTK_LIBDIR)
VTK_LIBS=-lvtkCommonCore-6.1\
 -lvtkImagingCore-6.1\
 -lvtkRenderingCore-6.1\
 -lvtkIOLegacy-6.1\
 -lvtkIOPLY-6.1\
 -lvtkIOXML-6.1\
 -lvtkInteractionStyle-6.1\
 -lvtkRenderingOpenGL-6.1\
 -lvtkRenderingFreeTypeOpenGL-6.1\
 -lvtkFiltersHybrid-6.1\
 -lvtkFiltersModeling-6.1\
 -lvtkGeovisCore-6.1\
 -lvtkRenderingOpenGL-6.1\
 -lvtkIOGeometry-6.1\
 -lvtkjsoncpp-6.1\
 -lvtkIOXMLParser-6.1\
 -lvtkexpat-6.1\
 -lvtkInfovisLayout-6.1\
 -lvtkInfovisCore-6.1\
 -lvtkViewsCore-6.1\
 -lvtkInteractionWidgets-6.1\
 -lvtkInteractionStyle-6.1\
 -lvtkFiltersHybrid-6.1\
 -lvtkFiltersModeling-6.1\
 -lvtkImagingHybrid-6.1\
 -lvtkIOImage-6.1\
 -lvtkIOCore-6.1\
 -lvtkpng-6.1\
 -lvtktiff-6.1\
 -lvtkjpeg-6.1\
 -lvtkmetaio-6.1\
 -lvtkDICOMParser-6.1\
 -lvtkImagingGeneral-6.1\
 -lvtkImagingSources-6.1\
 -lvtkRenderingAnnotation-6.1\
 -lvtkRenderingFreeType-6.1\
 -lvtkftgl-6.1\
 -lvtkzlib-6.1\
 -lvtkImagingColor-6.1\
 -lvtkRenderingVolume-6.1\
 -lvtkRenderingCore-6.1\
 -lvtkFiltersSources-6.1\
 -lvtkFiltersGeometry-6.1\
 -lvtkFiltersExtraction-6.1\
 -lvtkFiltersGeneral-6.1\
 -lvtkCommonComputationalGeometry-6.1\
 -lvtkFiltersCore-6.1\
 -lvtkFiltersStatistics-6.1\
 -lvtkImagingFourier-6.1\
 -lvtkImagingCore-6.1\
 -lvtkCommonExecutionModel-6.1\
 -lvtkCommonDataModel-6.1\
 -lvtkCommonMisc-6.1\
 -lvtkCommonSystem-6.1\
 -lvtkCommonTransforms-6.1\
 -lvtkCommonMath-6.1\
 -lvtkCommonCore-6.1\
 -lvtkfreetype-6.1\
 -lvtksys-6.1\
 -lvtkalglib-6.1\
 -lvtkproj4-6.1

ICET_LIBDIR=$(THIRDPARTY_DIR)/icet/1.0.0/$(ARCH)/lib
ICET_INCDIR=$(THIRDPARTY_DIR)/icet/1.0.0/$(ARCH)/include
ICET_LDFLAGS=-L$(ICET_LIBDIR)
ICET_LIBS=-licet -licet_mpi -licet_strategies

LIBSIM_INCDIR=$(HOME)/Development/install_cori_knl/2.13.0/linux-x86_64/libsim/V2/include
LIBSIM_LIBDIR=$(HOME)/Development/install_cori_knl/2.13.0/linux-x86_64/libsim/V2/lib
LIBSIM_LDFLAGS=-L$(LIBSIM_LIBDIR)
LIBSIM_LIBS=-lsimV2_static_par

LLVM_LIBDIR=$(THIRDPARTY_DIR)/llvm/4.0.0/$(ARCH)/lib
LLVM_LDFLAGS=-L$(LLVM_LIBDIR)
LLVM_LIBS=-lLLVM

GL_LIBDIR=$(THIRDPARTY_DIR)/openswr/17.0.6/$(ARCH)/lib
GL_LDFLAGS=$(LLVM_LDFLAGS)
GL_LIBS=$(GL_LIBDIR)/libOSMesa.a $(LLVM_LIBS)

CPPFLAGS=-I. -I$(LIBSIM_INCDIR) -I$(VTK_INCDIR) -DPARALLEL
LDFLAGS=$(LIBSIM_LDFLAGS) $(VTK_LDFLAGS) $(GL_LDFLAGS) $(ICET_LDFLAGS) $(MPI_LDFLAGS)
LIBS=$(LIBSIM_LIBS) $(VTK_LIBS) $(ICET_LIBS) $(MPI_LIBS) $(GL_LIBS) -ldl -lpthread -lz -lm

SRC=batch.c extract.c
OBJ=$(SRC:.c=.o)

all: batch_par

batch_par: $(OBJ)
	$(CXX) -o batch_par $(OBJ) $(CFLAGS) $(LDFLAGS) $(LIBS)

clean:
	$(RM) $(OBJ) batch_par

.c.o:
	$(CC) $(CFLAGS) $(CPPFLAGS) -c $<

Simulation Run

68 cores

Here is the sbatch script I used to submit the simulation on cori. Note that the -domains I,J,K numbers used must multiply out to the number of processors used for the job submission.

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 1
#SBATCH -t 00:10:00
#SBATCH -A m636
#SBATCH -C knl

cd $SLURM_SUBMIT_DIR
SIMDIR=$HOME/Development/MAIN/cori_build_static_icc/testlibsim
srun -n 68 $SIMDIR/batch_par -domains 17,4,1 -maxcycles 10 -export 0 -render 1 -format FieldViewXDB

The simulation wrote out 10 PNG files that I combined into a small animation here:

Libsim cori batch 68.gif

1224 cores

I ran another job at 1224 cores (18 nodes):

#!/bin/sh
#SBATCH -p debug
#SBATCH -N 18
#SBATCH -t 00:10:00
#SBATCH -A m636
#SBATCH -C knl

cd $SLURM_SUBMIT_DIR
SIMDIR=$HOME/Development/MAIN/cori_build_static_icc/testlibsim
date
srun -n 1224 $SIMDIR/batch_par -domains 17,9,8 -maxcycles 10 -export 0 -render 1 -format FieldViewXDB
date
Mon Aug  7 18:10:32 PDT 2017
Initialization time: 0.39311
Simulating time step: cycle=0, time=0
Saved batch0000.png
Simulating time step: cycle=1, time=0.314159
Saved batch0001.png
Simulating time step: cycle=2, time=0.628319
Saved batch0002.png
Simulating time step: cycle=3, time=0.942478
Saved batch0003.png
Simulating time step: cycle=4, time=1.25664
Saved batch0004.png
Simulating time step: cycle=5, time=1.5708
Saved batch0005.png
Simulating time step: cycle=6, time=1.88496
Saved batch0006.png
Simulating time step: cycle=7, time=2.19911
Saved batch0007.png
Simulating time step: cycle=8, time=2.51327
Saved batch0008.png
Simulating time step: cycle=9, time=2.82743
Saved batch0009.png
Mon Aug  7 18:11:44 PDT 2017

Libsim cori batch1224.gif

Lessons Learned

Here are a few things I learned.

CMake and MPI on Cray

CMake's FindMPI module doesn't like the Cray wrapper compiler for finding MPI if you happen to pass that for the CMAKE_C_COMPILER, CMAKE_CXX_COMPILER. You instead need to pass some more specific MPI variables to get CMake to include MPI in the include and link. For cori, I found it to look something like this:

#!/bin/sh

MPI_C_LIBRARIES="-Wl,--start-group /opt/cray/mpt/default/gni/mpich-gnu/5.1/lib/libmpich_gnu_51.a /opt/cray/pmi/default/lib64/libpmi.a /opt/cray/alps/default/lib64/libalpsutil.a /opt/cray/alps/default/lib64/libalpslli.a /opt/cray/alps/default/lib64/libalps.a /opt/cray/wlm_detect/default/lib64/libwlm_detect.a /opt/cray/ugni/default/lib64/libugni.a /opt/cray/udreg/default/lib64/libudreg.a /opt/cray/xpmem/default/lib64/libxpmem.a -lpthread /usr/lib64/librt.a -Wl,--end-group"

cmake -DCMAKE_C_COMPILER=/opt/gcc/5.2.0/bin/gcc \
-DCMAKE_CXX_COMPILER=/opt/gcc/5.2.0/bin/g++ \
-DMPI_C_INCLUDE_PATH=/opt/cray/mpt/7.3.1/gni/mpich-gnu/5.1/include \
-DMPI_C_LIBRARIES="$MPI_C_LIBRARIES" \
-DMPI_CXX_INCLUDE_PATH=/opt/cray/mpt/7.3.1/gni/mpich-gnu/5.1/include \
-DMPI_CXX_LIBRARIES="$MPI_C_LIBRARIES" \
/path/to/src

Detecting Libsim from CMake

Libsim doesn't currently bundle a find module for cmake. This isn't that big a deal when the normal dynamically-linked VisIt is used because you'd normally only link to libsimV2.a and libsimV2f.a, which are easy to include in a build. When VisIt is statically linked, the picture totally changes. All of a sudden, you must include the VTK, Ice-T, OpenGL, and anything else that VisIt needs to link statically. Static linking really would benefit from a FindLIBSIM.cmake module. I've taken a stab at one here so I could include a statically linked Libsim in one of my projects.

There were a couple issues:

  • VisIt's install does not install most 3rd party .a files, which makes it impossible to link to VTK in the VisIt installation.
  • Same thing with OpenGL when Mesa is used as OpenGL.

FindLIBSIM.cmake

find_package(LIBSIM REQUIRED)

add_library(libsim INTERFACE)
target_link_libraries(libsim INTERFACE ${LIBSIM_STATIC_PAR_LIBRARIES})
target_include_directories(libsim SYSTEM INTERFACE ${LIBSIM_INCLUDE_DIR})

I created a LIBSIMConfig.cmake file to be installed into the same lib directory as the Libsim .a files.

# LIBSIM_FOUND                - System has Libsim
# LIBSIM_INCLUDE_DIR          - The path to the Libsim header files
# LIBSIM_LIBRARY_DIR          - The path to the Libsim lib files
# LIBSIM_LIBRARIES            - The Libsim simV2.a file and 3rd party dependencies
# LIBSIM_FORTRAN_LIBRARIES    - The Libsim simV2.a, simV2f.a files and 3rd party dependencies
# LIBSIM_STATIC_SER_LIBRARIES - The Libsim simV2_static_ser.a file and 3rd party dependencies
# LIBSIM_STATIC_PAR_LIBRARIES - The Libsim simV2_static_par.a file and 3rd party dependencies (without MPI)

# Compute the installation prefix from this LIBSIMConfig.cmake file location.
get_filename_component(LIBSIM_INSTALL_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH)
set(LIBSIM_MODULES_DIR "${LIBSIM_INSTALL_PREFIX}/lib")
get_filename_component(VISIT_ARCH_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
get_filename_component(VISIT_ARCH_DIR "${VISIT_ARCH_DIR}" PATH)
get_filename_component(VISIT_ARCH_DIR "${VISIT_ARCH_DIR}" PATH)
get_filename_component(VISIT_ARCH_DIR "${VISIT_ARCH_DIR}" PATH)

set(LIBSIM_INSTALL_PREFIX ${VISIT_ARCH_DIR}/libsim/V2)
set(VISIT_ARCHIVE_DIR     ${VISIT_ARCH_DIR}/archives)

SET(LIBSIM_VERSION "2.11.0")

# NOTE: The engine-only build did not install any VisIt 3rd party libraries into the
#       VisIt installation. For now, we'll reference the libraries that were used to
#       build VisIt. However, to make it easier for users to statically link Libsim,
#       we'll need to install all those 3rd party .a's.

#if static and mesa found
set(LIBSIM_GL_LIBDIR /global/homes/w/whitlocb/Development/thirdparty_static_server/mesa/11.1.2/linux-x86_64_gcc-5.2/lib)
set(LIBSIM_GL_LIBRARIES ${LIBSIM_GL_LIBDIR}/libGL.a ${LIBSIM_GL_LIBDIR}/libglapi.a crypto dl)

#if static and icet found
set(LIBSIM_ICET_LIBDIR /global/homes/w/whitlocb/Development/thirdparty_static_server/icet/1.0.0/linux-x86_64_gcc-5.2/lib)
set(LIBSIM_ICET_LIBRARIES ${LIBSIM_ICET_LIBDIR}/libicet.a ${LIBSIM_ICET_LIBDIR}/libicet_mpi.a ${LIBSIM_ICET_LIBDIR}/libicet_strategies.a)

# VTK files
set(LIBSIM_VTK_LIBDIR /global/homes/w/whitlocb/Development/thirdparty_static_server/vtk/6.1.0/linux-x86_64_gcc-5.2/lib)
set(LIBSIM_VTK_LIBRARIES
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOLegacy-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOPLY-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOXML-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInteractionStyle-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingOpenGL-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingFreeTypeOpenGL-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersHybrid-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersModeling-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkGeovisCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingOpenGL-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOGeometry-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkjsoncpp-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOXMLParser-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkexpat-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInfovisLayout-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInfovisCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkViewsCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInteractionWidgets-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkInteractionStyle-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersHybrid-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersModeling-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingHybrid-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOImage-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkIOCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkpng-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtktiff-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkjpeg-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkmetaio-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkDICOMParser-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingGeneral-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingSources-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingAnnotation-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingFreeType-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkftgl-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkzlib-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingColor-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingVolume-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkRenderingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersSources-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersGeometry-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersExtraction-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersGeneral-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonComputationalGeometry-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkFiltersStatistics-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingFourier-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkImagingCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonExecutionModel-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonDataModel-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonMisc-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonSystem-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonTransforms-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonMath-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkCommonCore-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkfreetype-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtksys-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkalglib-6.1.a
 ${LIBSIM_VTK_LIBDIR}/libvtkproj4-6.1.a
 )
 
set(LIBSIM_FOUND 1)

set(LIBSIM_INCLUDE_DIR ${LIBSIM_INSTALL_PREFIX}/include)
set(LIBSIM_LIBRARY_DIR  ${LIBSIM_INSTALL_PREFIX}/lib)

set(LIBSIM_THIRDPARTY_LIBRARIES ${LIBSIM_VTK_LIBRARIES} ${LIBSIM_ICET_LIBRARIES} ${LIBSIM_GL_LIBRARIES} z)

set(LIBSIM_LIBRARIES  ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2.a ${LIBSIM_THIRDPARTY_LIBRARIES})
set(LIBSIM_FORTRAN_LIBRARIES ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2.a ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2f.a ${LIBSIM_THIRDPARTY_LIBRARIES})
set(LIBSIM_STATIC_SER_LIBRARIES  ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2_static_ser.a ${LIBSIM_THIRDPARTY_LIBRARIES})
set(LIBSIM_STATIC_PAR_LIBRARIES  ${LIBSIM_INSTALL_PREFIX}/lib/libsimV2_static_par.a ${LIBSIM_THIRDPARTY_LIBRARIES})

MESSAGE(STATUS "LIBSIM_FOUND = ${LIBSIM_FOUND}")
MESSAGE(STATUS "LIBSIM_VERSION = ${LIBSIM_VERSION}")
MESSAGE(STATUS "LIBSIM_INCLUDE_DIR = ${LIBSIM_INCLUDE_DIR}")
MESSAGE(STATUS "LIBSIM_LIBRARY_DIR = ${LIBSIM_LIBRARY_DIR}")
MESSAGE(STATUS "LIBSIM_LIBRARIES = ${LIBSIM_LIBRARIES}")
MESSAGE(STATUS "LIBSIM_FORTRAN_LIBRARIES = ${LIBSIM_FORTRAN_LIBRARIES}")
MESSAGE(STATUS "LIBSIM_STATIC_SER_LIBRARIES = ${LIBSIM_STATIC_SER_LIBRARIES}")
MESSAGE(STATUS "LIBSIM_STATIC_PAR_LIBRARIES = ${LIBSIM_STATIC_PAR_LIBRARIES}")

If the FindLIBSIM.cmake file is included into your CMakeLists.txt, and you define LIBSIM_DIR=/path/to/dir/containing/LIBSIMConfig.cmake on the cmake command line then cmake should locate Libsim and set up the Libsim variables. From there, you can add code like this in your CMakeLists.txt:

 target_include_directories(myapp SYSTEM PUBLIC ${LIBSIM_INCLUDE_DIR})
 target_link_libraries(myapp PRIVATE libsim)