Audit the compiler flags used to build on gadi with the Intel compiler

[harshula]

Document compiler flags of the following components:

• oasis3-mct
• libaccessom2
• cice5
• mom5

[harshula]

oasis3-mct
util/make_dir/make.nci:

#
# Include file for OASIS3 Makefile for a Linux system using 
# Portland Group Fortran Compiler and MPICH
#
###############################################################################
#
# CHAN	: communication technique used in OASIS3 (MPI1/MPI2/NONE)
CHAN            = MPI1
#
# Paths for libraries, object files and binaries
#
# COUPLE	: path for oasis3-mct main directory
COUPLE=$(OASIS_HOME)
# ARCHDIR       : directory created when compiling
ARCHDIR= $(COUPLE)/compile_oa3-mct

#
# NETCDF library
NETCDF_INCLUDE = $(NETCDF_ROOT)/include
NETCDF_LIBRARY = -L$(NETCDF_ROOT)/lib -lnetcdf -lnetcdff
#
# Compiling and other commands
MAKE        = /usr/bin/make
F90         = mpifort
CC          = mpicc
F           = $(F90)
f90         = $(F90)
f           = $(F90)

LD          = $(F90)
AR          = ar
ARFLAGS     = -rvD

# -g is necessary in F90FLAGS and LDFLAGS for pgf90 versions lower than 6.1
# For compiling in double precision, put -r8
# For compiling in single precision, remove -r8 and add -Duse_realtype_single
NCI_INTEL_FLAGS = -r8 -i4 -traceback -fpe0 -convert big_endian -fno-alias -ip -check noarg_temp_created
NCI_REPRO_FLAGS = -fp-model precise -fp-model source -align all
ifeq ($(DEBUG), yes)
    NCI_DEBUG_FLAGS = -g3 -O0 -fpe0 -no-vec -debug all -check all -no-vec
    F90FLAGS_1      = $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_DEBUG_FLAGS)
    CPPDEF          = -Duse_netCDF -Duse_comm_$(CHAN) -DTREAT_OVERLAY -DDEBUG -D__VERBOSE
    MCT_FCFLAGS     = $(NCI_REPRO_FLAGS) $(NCI_DEBUG_FLAGS) -ip
else
    NCI_OPTIM_FLAGS = -g3 -O2 -axCORE-AVX2 -debug all -check none -qopt-report=5 -qopt-report-annotate
    F90FLAGS_1      = $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_OPTIM_FLAGS)
    CPPDEF          = -Duse_netCDF -Duse_comm_$(CHAN) -DTREAT_OVERLAY
    MCT_FCFLAGS     = $(NCI_REPRO_FLAGS) $(NCI_OPTIM_FLAGS) -ip
endif
f90FLAGS_1  = $(F90FLAGS_1)
FFLAGS_1    = $(F90FLAGS_1)
fFLAGS_1    = $(F90FLAGS_1)
CCFLAGS_1   = 
LDFLAGS     = 

#
###################
#
# Additional definitions that should not be changed
#
FLIBS		= $(NETCDF_LIBRARY)
# BINDIR        : directory for executables
BINDIR          = $(ARCHDIR)/bin
# LIBBUILD      : contains a directory for each library
LIBBUILD        = $(ARCHDIR)/build/lib
# INCPSMILE     : includes all *o and *mod for each library
INCPSMILE       = -I$(LIBBUILD)/psmile.$(CHAN) -I$(LIBBUILD)/pio  -I$(LIBBUILD)/mct 

F90FLAGS  = $(F90FLAGS_1) $(INCPSMILE) $(CPPDEF) -I$(NETCDF_INCLUDE)
f90FLAGS  = $(f90FLAGS_1) $(INCPSMILE) $(CPPDEF) -I$(NETCDF_INCLUDE)
FFLAGS    = $(FFLAGS_1) $(INCPSMILE) $(CPPDEF) -I$(NETCDF_INCLUDE)
fFLAGS    = $(fFLAGS_1) $(INCPSMILE) $(CPPDEF) -I$(NETCDF_INCLUDE)
CCFLAGS   = $(CCFLAGS_1) $(INCPSMILE) $(CPPDEF) -I$(NETCDF_INCLUDE)
#
#############################################################################

[harshula]

libaccessom2
CMakeLists.txt:

# compiler flags for gfortran
if(CMAKE_Fortran_COMPILER_ID MATCHES GNU)
  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -std=f2008 -Wall -fdefault-real-8 -ffpe-trap=invalid,zero,overflow")
  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -g -pg -fbounds-check -fbacktrace")
  set(CMAKE_Fortran_FLAGS_RELEASE "-O3")
endif()

# compiler flags for ifort
if(CMAKE_Fortran_COMPILER_ID MATCHES Intel)
  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -fpe0 -fp-model precise -fp-model source -align all -traceback")
  set(CMAKE_Fortran_FLAGS_DEBUG "-g3 -O0 -check all")
  set(CMAKE_Fortran_FLAGS_RELEASE "-g3 -O2 -axCORE-AVX2 -debug all -check none -qopt-report=5 -qopt-report-annotate")
endif()

[harshula]

cice5
bld/Macros.nci

#==============================================================================
# Makefile macros for gadi.nci.org.au
#==============================================================================

INCLDIR    := -I.
SLIBS      :=
ULIBS      :=
CPP        := cpp
FC         := mpifort

CPPFLAGS   := -P -traditional
CPPDEFS    := -DLINUX -DPAROPT
CFLAGS     := -c -O2
FIXEDFLAGS := -132
FREEFLAGS  :=

NCI_INTEL_FLAGS := -r8 -i4 -traceback -w -fpe0 -ftz -convert big_endian -assume byterecl -check noarg_temp_created
NCI_REPRO_FLAGS := -fp-model precise -fp-model source -align all
ifeq ($(DEBUG), 1)
    NCI_DEBUG_FLAGS := -g3 -O0 -debug all -check all -no-vec -assume nobuffered_io
    FFLAGS          := $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_DEBUG_FLAGS)
    CPPDEFS         := $(CPPDEFS) -DDEBUG=$(DEBUG)
else
    NCI_OPTIM_FLAGS := -g3 -O2 -axCORE-AVX2 -debug all -check none -qopt-report=5 -qopt-report-annotate -assume buffered_io
    FFLAGS          := $(NCI_INTEL_FLAGS) $(NCI_REPRO_FLAGS) $(NCI_OPTIM_FLAGS)
endif

MOD_SUFFIX := mod
LD         := $(FC)
LDFLAGS    := $(FFLAGS) -v

CPPDEFS :=  $(CPPDEFS) -DNXGLOB=$(NXGLOB) -DNYGLOB=$(NYGLOB) \
            -DNUMIN=$(NUMIN) -DNUMAX=$(NUMAX) \
            -DTRAGE=$(TRAGE) -DTRFY=$(TRFY) -DTRLVL=$(TRLVL) \
            -DTRPND=$(TRPND) -DNTRAERO=$(NTRAERO) -DTRBRI=$(TRBRI) \
            -DNBGCLYR=$(NBGCLYR) -DTRBGCS=$(TRBGCS) \
            -DNICECAT=$(NICECAT) -DNICELYR=$(NICELYR) \
            -DNSNWLYR=$(NSNWLYR) \
            -DBLCKX=$(BLCKX) -DBLCKY=$(BLCKY) -DMXBLCKS=$(MXBLCKS)

ifeq ($(COMMDIR), mpi)
   SLIBS   :=  $(SLIBS) -lmpi
endif

ifeq ($(DITTO), yes)
   CPPDEFS :=  $(CPPDEFS) -DREPRODUCIBLE
endif
ifeq ($(BARRIERS), yes)
   CPPDEFS :=  $(CPPDEFS) -Dgather_scatter_barrier
endif

ifeq ($(IO_TYPE), netcdf)
   CPPDEFS :=  $(CPPDEFS) -Dncdf
   INCLDIR := $(INCLDIR) -I$(NETCDF_ROOT)/include
   SLIBS   := $(SLIBS) -L$(NETCDF_ROOT)/lib -lnetcdf -lnetcdff
endif

ifeq ($(IO_TYPE), pio)
   CPPDEFS :=  $(CPPDEFS) -Dncdf -DPIO
   INCLDIR := $(INCLDIR) -I$(NETCDF_ROOT)/include
   SLIBS   := $(SLIBS) -L$(NETCDF_ROOT)/lib -lnetcdf -lnetcdff
   SLIBS   := $(SLIBS) -L$(SRCDIR)/ParallelIO/build/lib/ -lpiof -lpioc -Wl,-rpath=$(SRCDIR)/ParallelIO/build/lib/
endif

ifeq ($(USE_ESMF), yes)
   CPPDEFS :=  $(CPPDEFS) -Duse_esmf
   INCLDIR :=  $(INCLDIR) -I ???
   SLIBS   :=  $(SLIBS) -L ??? -lesmf -lcprts -lrt -ldl
endif

ifeq ($(AusCOM), yes)
   CPPDEFS := $(CPPDEFS) -DAusCOM -Dcoupled
   INCLDIR := $(INCLDIR) $(CPL_INCS) $(LIBAUSCOM_INCS)
   SLIBS   := $(SLIBS) $(CPLLIBS)
endif

ifeq ($(UNIT_TESTING), yes)
   CPPDEFS := $(CPPDEFS) -DUNIT_TESTING
endif
ifeq ($(ACCESS), yes)
   CPPDEFS := $(CPPDEFS) -DACCESS
endif
# standalone CICE with AusCOM mods
ifeq ($(ACCICE), yes)
   CPPDEFS := $(CPPDEFS) -DACCICE
endif
# no MOM just CICE+UM
ifeq ($(NOMOM), yes)
   CPPDEFS := $(CPPDEFS) -DNOMOM
endif
ifeq ($(OASIS3_MCT), yes)
   CPPDEFS := $(CPPDEFS) -DOASIS3_MCT
endif

[harshula]

mom5
bin/mkmf.template.nci

# Template for the NCI (nf.nci.org.au) machines. Uses intel compiler and OpenMPI.
# typical use with mkmf
# mkmf -t template.ifc -c"-Duse_libMPI -Duse_netCDF" path_names /usr/local/include
############
# commands #
############
ifeq ($(VTRACE), yes)
    FC := mpifort-vt
    LD := mpifort-vt
else
    FC := mpifort
    LD := mpifort
endif

CC := mpicc

#########
# flags #
#########
VERBOSE :=
OPT := on

MAKEFLAGS += -j

INCLUDE   := -I$(NETCDF_ROOT)/include

ifneq ($(LIBACCESSOM2_ROOT),)
INCLUDE  += -I$(LIBACCESSOM2_ROOT)/oasis3-mct/Linux/build/lib/psmile.MPI1 \
            -I$(LIBACCESSOM2_ROOT)/oasis3-mct/Linux/build/lib/mct \
            -I$(LIBACCESSOM2_ROOT)/build/include
endif

ifneq ($(OASIS_ROOT),)
INCLUDE  += -I$(OASIS_ROOT)/Linux/build/lib/psmile.MPI1 \
            -I$(OASIS_ROOT)/Linux/build/lib/pio \
            -I$(OASIS_ROOT)/Linux/build/lib/mct
endif

FPPFLAGS := -fpp -Wp,-w $(INCLUDE)
FFLAGS := -fno-alias -safe-cray-ptr -fpe0 -ftz -assume byterecl -i4 -r8 -traceback -nowarn -check noarg_temp_created -assume nobuffered_io -convert big_endian -grecord-gcc-switches -align all
FFLAGS_OPT := -g3 -O2 -xCORE-AVX2 -debug all -check none
FFLAGS_REPORT := -qopt-report=5 -qopt-report-annotate
FFLAGS_DEBUG := -g3 -O0 -debug all -check -check noarg_temp_created -check nopointer -warn -warn noerrors -ftrapuv
FFLAGS_REPRO := -fp-model precise -fp-model source -align all
FFLAGS_VERBOSE := -v -V -what

CFLAGS := -D__IFC $(INCLUDE)
CFLAGS_OPT := -O2 -debug minimal -xCORE-AVX2
CFLAGS_REPORT := -qopt-report=5 -qopt-report-annotate
CFLAGS_DEBUG := -O0 -g -ftrapuv -traceback
CFLAGS_REPRO := -fp-model precise -fp-model source

LDFLAGS :=
LDFLAGS_VERBOSE := -Wl,-V,--verbose,-cref,-M

ifneq ($(REPRO),)
CFLAGS += $(CFLAGS_REPRO)
FFLAGS += $(FFLAGS_REPRO)
endif

ifneq ($(DEBUG),)
CFLAGS += $(CFLAGS_DEBUG)
FFLAGS += $(FFLAGS_DEBUG)
else
CFLAGS += $(CFLAGS_OPT)
FFLAGS += $(FFLAGS_OPT)
endif

ifneq ($(VERBOSE),)
CFLAGS += $(CFLAGS_VERBOSE)
FFLAGS += $(FFLAGS_VERBOSE)
LDFLAGS += $(LDFLAGS_VERBOSE)
endif

ifneq ($(REPORT),)
CFLAGS += $(CFLAGS_REPORT)
FFLAGS += $(FFLAGS_REPORT)
endif

LIBS := -L$(NETCDF_ROOT)/lib -lnetcdf -lnetcdff \

ifneq ($(OASIS_ROOT),)
LIBS += -L$(OASIS_ROOT)/Linux/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip
endif

ifneq ($(LIBACCESSOM2_ROOT),)
LIBS += -L$(LIBACCESSOM2_ROOT)/build/lib -laccessom2
endif

LDFLAGS += $(LIBS)

#---------------------------------------------------------------------------
# you should never need to change any lines below.

# see the MIPSPro F90 manual for more details on some of the file extensions
# discussed here.
# this makefile template recognizes fortran sourcefiles with extensions
# .f, .f90, .F, .F90. Given a sourcefile <file>.<ext>, where <ext> is one of
# the above, this provides a number of default actions:

# make <file>.opt	create an optimization report
# make <file>.o		create an object file
# make <file>.s		create an assembly listing
# make <file>.x		create an executable file, assuming standalone
#			source
# make <file>.i		create a preprocessed file (for .F)
# make <file>.i90	create a preprocessed file (for .F90)

# The macro TMPFILES is provided to slate files like the above for removal.

RM := rm -f
SHELL := /bin/csh -f
TMPFILES := .*.m *.B *.L *.i *.i90 *.l *.s *.mod *.opt

.SUFFIXES: .F .F90 .H .L .T .f .f90 .h .i .i90 .l .o .s .opt .x

.f.L:
	$(FC) $(FFLAGS) -c -listing $*.f
.f.opt:
	$(FC) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.f
.f.l:
	$(FC) $(FFLAGS) -c $(LIST) $*.f
.f.T:
	$(FC) $(FFLAGS) -c -cif $*.f
.f.o:
	$(FC) $(FFLAGS) -c $*.f
.f.s:
	$(FC) $(FFLAGS) -S $*.f
.f.x:
	$(FC) $(FFLAGS) -o $*.x $*.f *.o $(LDFLAGS)
.f90.L:
	$(FC) $(FFLAGS) -c -listing $*.f90
.f90.opt:
	$(FC) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.f90
.f90.l:
	$(FC) $(FFLAGS) -c $(LIST) $*.f90
.f90.T:
	$(FC) $(FFLAGS) -c -cif $*.f90
.f90.o:
	$(FC) $(FFLAGS) -c $*.f90
.f90.s:
	$(FC) $(FFLAGS) -c -S $*.f90
.f90.x:
	$(FC) $(FFLAGS) -o $*.x $*.f90 *.o $(LDFLAGS)
.F.L:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -listing $*.F
.F.opt:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.F
.F.l:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $(LIST) $*.F
.F.T:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -cif $*.F
.F.f:
	$(FC) $(CPPDEFS) $(FPPFLAGS) -EP $*.F > $*.f
.F.i:
	$(FC) $(CPPDEFS) $(FPPFLAGS) -P $*.F
.F.o:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $*.F
.F.s:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -S $*.F
.F.x:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -o $*.x $*.F *.o $(LDFLAGS)
.F90.L:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -listing $*.F90
.F90.opt:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -opt_report_level max -opt_report_phase all -opt_report_file $*.opt $*.F90
.F90.l:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $(LIST) $*.F90
.F90.T:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -cif $*.F90
.F90.f90:
	$(FC) $(CPPDEFS) $(FPPFLAGS) -EP $*.F90 > $*.f90
.F90.i90:
	$(FC) $(CPPDEFS) $(FPPFLAGS) -P $*.F90
.F90.o:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c $*.F90
.F90.s:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -c -S $*.F90
.F90.x:
	$(FC) $(CPPDEFS) $(FPPFLAGS) $(FFLAGS) -o $*.x $*.F90 *.o $(LDFLAGS)

[access-hive-bot]

This issue has been mentioned on ACCESS Hive Community Forum. There might be relevant details there:

https://forum.access-hive.org.au/t/audit-the-compiler-flags-used-to-build-access-om2-on-gadi-with-the-intel-compiler/437/1

[aidanheerdegen]

The -opt_report_level flags should never been kept in the production build. They produce a huge amount of profiling information that isn't relevant unless you're interested in profiling the code.

[aidanheerdegen]

Note that I am not volunteering to do this, but it would be good to extract out the exact flags used in each model and chuck it in a table or similar to be able to compare them across models.

[aekiss]

might be easiest to build access-om2 and copy from what it echoes to the terminal

[harshula]

Notes

https://www.intel.com/content/www/us/en/develop/documentation/fortran-compiler-oneapi-dev-guide-and-reference/top.html

https://www.nas.nasa.gov/hecc/support/kb/recommended-intel-compiler-debugging-options_92.html
https://www.nas.nasa.gov/hecc/support/kb/recommended-compiler-options_99.html

[harshula]

Notes

https://www.intel.com/content/www/us/en/develop/documentation/fortran-compiler-oneapi-dev-guide-and-reference/top/compiler-reference/compiler-options/code-generation-options/march.html

march
...
Default
pentium4
If no architecture option is specified, value pentium4 is used by the compiler to generate code.
...
Description
This option tells the compiler to generate code for processors that support certain features. If you specify both the -ax and -march options, the compiler will not generate Intel-specific instructions.
Options -x and -march are mutually exclusive. If both are specified, the compiler uses the last one specified and generates a warning.
Specifying -march=pentium4 sets -mtune=pentium4.
For compatibility, a number of historical processor values are also supported, but the generated code will not differ from the default.

[harshula]

Notes
-no-vec does not appear to override vectorization via -march. Worth further investigation.

[harshula]

Notes
Using the argument target=x86_64 when building a Spack package causes SPACK_TARGET_ARGS='-march=pentium4 -mtune=generic'. This appears to override -axCORE-AVX2. Worth further investigation. Setting no target on Gadi causes SPACK_TARGET_ARGS='-march=cascadelake -mtune=cascadelake'

[harshula]

Notes
Instruction Set support on various architectures as understood by Spack:
https://github.com/spack/spack/blob/develop/lib/spack/external/archspec/json/cpu/microarchitectures.json