Merge pull request #235 from bonachea/network-conduits

Support distributed-memory (network) configurations

Author: bonachea

.github/workflows/build.yml

@@ -8,45 +8,70 @@ defaults:
 
 jobs:
   build:
-    name: ${{ matrix.compiler }}-${{ matrix.version }} (${{ matrix.os }})
+    name: ${{ matrix.compiler }}-${{ matrix.version }} ${{ matrix.network }} (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [macos-13, macos-14, macos-15, ubuntu-24.04]
         compiler: [ gfortran ]
         version: [ 13, 14 ]
+        network: [ smp ]
         exclude:
           - os: macos-15 # gcc-13 broken on macos-15
+            compiler: gfortran
             version: 13
+            network: smp
         include:
+        # --- flang coverage ---
           - os: macos-13
             compiler: flang
             version: 20
+            network: smp
           - os: macos-14
             compiler: flang
             version: 20
+            network: smp
           - os: macos-15
             compiler: flang
             version: 20
+            network: smp
           - os: ubuntu-24.04
             compiler: flang
             version: 20
+            network: smp
             # https://hub.docker.com/r/phhargrove/llvm-flang/tags
             container: phhargrove/llvm-flang:20.1.0-1
-            SUBJOB_PREFIX: GASNET_PSHM_NODES=2
           - os: ubuntu-24.04
             compiler: flang
             version: 19
+            network: smp
             FFLAGS: -mmlir -allow-assumed-rank
             # https://hub.docker.com/r/phhargrove/llvm-flang/tags
             container: phhargrove/llvm-flang:19.1.1-1
-            SUBJOB_PREFIX: GASNET_PSHM_NODES=2
 #          - os: ubuntu-24.04
 #            compiler: flang
 #            version: new
+#            network: smp
 #            container: gmao/llvm-flang:latest
-#            SUBJOB_PREFIX: GASNET_PSHM_NODES=2
+        # --- udp coverage for selected configs ---
+          - os: macos-15
+            compiler: gfortran
+            version: 14
+            network: udp
+          - os: ubuntu-24.04
+            compiler: gfortran
+            version: 14
+            network: udp
+          - os: macos-15
+            compiler: flang
+            version: 20
+            network: udp
+          - os: ubuntu-24.04
+            compiler: flang
+            version: 20
+            network: udp
+            container: phhargrove/llvm-flang:20.1.0-1
 
     container:
       image: ${{ matrix.container }}
@@ -56,8 +81,9 @@ jobs:
       FFLAGS: ${{ matrix.FFLAGS }}
       PREFIX: install
       GASNET_CONFIGURE_ARGS: --enable-rpath --enable-debug
-      GASNET_PSHM_NODES: 8
-      SUBJOB_PREFIX: ${{ matrix.SUBJOB_PREFIX }}
+      GASNET_SPAWN_VERBOSE: 1
+      GASNET_SPAWNFN: L
+      CAF_IMAGES: 8
 
     steps:
     - name: Set gfortran variables
@@ -76,6 +102,18 @@ jobs:
         echo "CC=clang" >> "$GITHUB_ENV"
         echo "CXX=clang++" >> "$GITHUB_ENV"
 
+    - name: Set Caffeine variables
+      run: |
+        set -x
+        # docker instances cannot handle high levels of subjob parallelism
+        if test -n "${{ matrix.container }}"; then \
+          echo "SUBJOB_PREFIX=CAF_IMAGES=2" >> "$GITHUB_ENV" ; \
+        fi
+        # disable shared-memory bypass with network=udp to simulate multi-node runs
+        if test "${{ matrix.network }}" = "udp"; then \
+          echo "GASNET_SUPERNODE_MAXSIZE=1" >> "$GITHUB_ENV" ; \
+        fi
+
     - name: Checkout code
       uses: actions/checkout@v1
 
@@ -116,9 +154,10 @@ jobs:
       if: ${{ contains(matrix.os, 'macos') && matrix.os != 'macos-13' }} 
       run: |
         set -x
-        curl --retry 5 -LOsS https://github.com/fortran-lang/fpm/releases/download/v0.11.0/fpm-0.11.0.F90
+        export FPM_VERSION=0.12.0
+        curl --retry 5 -LOsS https://github.com/fortran-lang/fpm/releases/download/v$FPM_VERSION/fpm-$FPM_VERSION.F90
         mkdir fpm-temp
-        gfortran-14 -o fpm-temp/fpm fpm-0.11.0.F90
+        gfortran-14 -o fpm-temp/fpm fpm-$FPM_VERSION.F90
         echo "PATH=${PWD}/fpm-temp:${PATH}" >> "$GITHUB_ENV"
 
     - name: Version info
@@ -137,11 +176,11 @@ jobs:
         for var in FC CC CXX FFLAGS CPPFLAGS CFLAGS LDFLAGS LIBS GASNET_CONFIGURE_ARGS ; do \
             eval echo "$var=\$$var"; done
         set -x
-        ./install.sh --prefix=${PREFIX} --verbose
+        ./install.sh --prefix=${PREFIX} --network=${{ matrix.network }} --verbose
 
     - name: Run examples
       run: |
-        echo GASNET_PSHM_NODES=${GASNET_PSHM_NODES}
+        echo CAF_IMAGES=${CAF_IMAGES}
         set -x
         ./build/run-fpm.sh run --verbose --example hello
         ./build/run-fpm.sh run --verbose --example stop_with_no_code
@@ -151,10 +190,10 @@ jobs:
     - name: Run unit tests
       run: |
         echo SUBJOB_PREFIX=${SUBJOB_PREFIX}
-        while (( GASNET_PSHM_NODES > 0 )); do \
-          echo GASNET_PSHM_NODES=${GASNET_PSHM_NODES} ; \
+        while (( CAF_IMAGES > 0 )); do \
+          echo CAF_IMAGES=${CAF_IMAGES} ; \
           ( set -x ; ./build/run-fpm.sh test --verbose -- -d ) ; \
           sleep 1 ; \
-          GASNET_PSHM_NODES=$(( GASNET_PSHM_NODES / 2 )) ; \
+          CAF_IMAGES=$(( CAF_IMAGES / 2 )) ; \
         done
 

CONTRIBUTING.md

@@ -10,11 +10,11 @@ If you encounter problems or limitations when installing or using Caffeine, plea
  2. Email the Caffeine [mailing list](mailto:[email protected]) for advice.
  3. If steps 1 or 2 do not resolve the problem, please file a [new issue] including
     - [ ] The Fortran compiler and compiler version used with Caffeine,
-    - [ ] The complete output of the build command,
+    - [ ] The complete output of the install and build commands run with `--verbose` argument,
     - [ ] The Caffeine version number or commit hash,
     - [ ] Any conditions required to reproduce the problem such as
       - [ ] The output of `uname -a` showing the operating system (OS), OS version, and processor architecture,
-      - [ ] The number of images executed (e.g., the output of `echo $GASNET_PSHM_NODES`),
+      - [ ] The number of images executed (e.g., the output of `echo $CAF_IMAGES`),
       - [ ] The command used to run your program (e.g., `./build/run-fpm.sh run`), and
       - [ ] A minimal reproducer: if possible, fewer than 50 lines demonstrating an issue.
 

README.md

@@ -88,14 +88,25 @@ Caffeine leverages the following non-parallel features of Fortran to simplify th
 
 Download, build, and run an example
 -----------------------------------
+Here is an outline of the basic commands used to build Caffeine and run an example:
+
 ```
 git clone https://github.com/BerkeleyLab/caffeine.git
 cd caffeine
-./install.sh
-export GASNET_PSHM_NODES=8
-FC=<Fortran-compiler-path> CC=<C-compiler-path> CXX=<C++-compiler-path> ./build/run-fpm.sh run --example hello
+env FC=<Fortran-compiler> CC=<C-compiler> CXX=<C++-compiler> ./install.sh <options>
+env CAF_IMAGES=8 ./build/run-fpm.sh run --example hello
 ```
 
+The provided compilers MUST be "compatible": for the best experience you are
+HIGHLY recommended to specify the language frontends provided by a single version
+of a given compiler suite installation. The C++ compiler is optional for
+single-node deployments (and can be disabled using command-line option `--without-cxx`), 
+but C++ is required for some network backends.
+
+The `install.sh` recognizes a number of command-line options and environment variables to
+customize behavior for your system. See the output of `./install.sh --help` for full documentation.
+
+
 Example Usage
 -------------
 The Caffeine parallel runtime is intended as an embedded compilation target
@@ -115,6 +126,31 @@ Run tests
 ./build/run-fpm.sh test
 ```
 
+Recognized Environment Variables
+--------------------------------
+
+The following environment variables control the execution of the `fpm`-driven Caffeine unit test suite:
+
+* `CAF_IMAGES`: integer that indicates the number of images to run
+* `SUBJOB_PREFIX`: command prefix to use for recursive `fpm` invocations in the test suite. 
+   Set `SUBJOB_PREFIX=skip` to disable such invocations (recommended for distributed-memory systems).
+
+The following environment variables control the behavior of the Caffeine library:
+
+* `CAF_HEAP_SIZE=128MB`: set the size of the shared-memory heap used for coarray storage, defaults to 128 MiB
+* `CAF_COMP_FRAC=0.10`: set the fraction of the shared-memory heap reserved for non-symmetric allocation, defaults to 10%
+
+Caffeine is built atop the [GASNet-EX] exascale networking middleware, which has its own
+set of environment variable knobs to control network-level behavior. 
+Here are *a few* of the most useful GASNet knobs:
+
+* `GASNET_VERBOSEENV=1`: enable console output of all the envvar settings affecting GASNet operation
+* `GASNET_SPAWN_VERBOSE=1`: enable verbose console output of parallel job-spawning steps
+* `GASNET_BACKTRACE=1`: enable automatic backtrace upon fatal errors
+* `GASNET_SSH_SERVERS="host1 host2"`: space-deliminted list of hostnames for distributed-memory job launch using the ssh-spawner
+
+See [GASNet documentation](https://gasnet.lbl.gov/dist-ex/README) for full details on all settings.
+
 Implementation Status
 --------------------
 

install.sh

@@ -12,6 +12,13 @@ USAGE:
 
  --help             Display this help text
  --prefix=PREFIX    Install library into 'PREFIX' directory
+ --network=<NET>    Build Caffeine to target given GASNet network conduit. 
+                    <NET> should be one of:
+                      smp: single-node shared-memory conduit (default)
+                      udp: portable UDP/IP (for Ethernet networks)
+                      ibv: InfiniBand IB Verbs
+                      ofi: OpenFabrics Interfaces
+                      ucx: Unified Communication X
  --prereqs          Display a list of prerequisite software.
                     Default prefix='\$HOME/.local/bin'
  --verbose          Show verbose build commands
@@ -43,6 +50,7 @@ EOF
 GCC_VERSION=${GCC_VERSION:=14}
 GASNET_VERSION="stable"
 VERBOSE=""
+GASNET_CONDUIT="${GASNET_CONDUIT:-smp}"
 
 list_prerequisites()
 {
@@ -82,6 +90,16 @@ while [ "$1" != "" ]; do
         --prefix)
             PREFIX=$VALUE
             ;;
+        --network)
+            GASNET_CONDUIT=$(tr '[:upper:]' '[:lower:]' <<< $VALUE)
+            case $GASNET_CONDUIT in
+              smp|udp|mpi|ibv|ofi|ucx) ;;
+              *) 
+                 echo "ERROR: Unrecognized --network=$GASNET_CONDUIT"
+                 print_usage_info
+                 exit 1
+            esac
+            ;;
         --verbose)
             VERBOSE="--verbose"
             set -x
@@ -304,8 +322,6 @@ EOF
   printf "Is it ok to download and install $1? [yes] "
 }
 
-# TODO: Expand this to other GASNet conduits (issue #66)
-GASNET_CONDUIT=smp
 pkg="gasnet-$GASNET_CONDUIT-seq"
 export PKG_CONFIG_PATH
 
@@ -337,8 +353,8 @@ if ! $PKG_CONFIG $pkg ; then
       cmd="$cmd --enable-$GASNET_CONDUIT"
       cmd="$cmd --enable-seq --disable-par --disable-parsync"
       cmd="$cmd --disable-segment-everything"
-      # TEMPORARY: disable MPI compatibility until Caffeine supports distributed conduits
-      cmd="$cmd --without-mpicc"
+      # TEMPORARY: disable MPI compatibility until we figure out how to support in fpm
+      cmd="$cmd --disable-mpi-compat"
       eval $cmd
       $MAKE -j 8 all
       $MAKE -j 8 install
@@ -366,6 +382,7 @@ GASNET_CPPFLAGS="`$PKG_CONFIG $pkg --variable=GASNET_CPPFLAGS`"
 #       in the directory path, and assumes that the first directory returned
 #       by pkg-config contains the GASNet lib directory
 GASNET_LIBDIR="$(echo $GASNET_LIBS | awk '{print $1};')"
+GASNET_LIBDIR=${GASNET_LIBDIR#-L}
 case "$GASNET_LIBDIR" in
   *spack* )
 	cat << EOF
@@ -378,7 +395,12 @@ EOF
     exit 1
     ;;
   * )
-    ;; # Do nothing otherwise 
+    GASNET_PREFIX=$(dirname $GASNET_LIBDIR)
+    if [ ! -r "$GASNET_PREFIX/include/gasnetex.h" ] ; then
+      echo "ERROR: Failed to detect GASNet install prefix from $GASNET_LIBS"
+      exit 1
+    fi
+    ;; 
 esac
 
 # Strip compiler flags
@@ -395,6 +417,9 @@ echo "# DO NOT EDIT OR COMMIT -- Created by caffeine/install.sh" > build/fpm.tom
 cp manifest/fpm.toml.template build/fpm.toml
 GASNET_LIB_LOCATIONS=`echo $GASNET_LIBS | awk '{locs=""; for(i = 1; i <= NF; i++) if ($i ~ /^-L/) {locs=(locs " " $i);}; print locs; }'`
 GASNET_LIB_NAMES=`echo $GASNET_LIBS | awk '{names=""; for(i = 1; i <= NF; i++) if ($i ~ /^-l/) {names=(names " " $i);}; print names; }' | sed 's/-l//g'`
+if [[ $GASNET_CONDUIT == "udp" ]] ; then
+  GASNET_LIB_NAMES+=" stdc++" # udp-conduit requires C++ libraries
+fi
 FPM_TOML_LINK_ENTRY="link = [\"$(echo ${GASNET_LIB_NAMES} | sed 's/ /", "/g')\"]"
 echo "${FPM_TOML_LINK_ENTRY}" >> build/fpm.toml
 ln -f -s build/fpm.toml
@@ -428,9 +453,30 @@ if ! [[ "$user_compiler_flags " =~ -[DU]ASSERTIONS[=\ ] ]] ; then
   compiler_flag+=" -DASSERTIONS"
 fi
 
+GASNET_CONDUIT_UPPER=$(tr '[:lower:]' '[:upper:]' <<<$GASNET_CONDUIT)
+compiler_flag+=" -DCAF_NETWORK_$GASNET_CONDUIT_UPPER"
+
 # Should come last to allow command-line overrides
 compiler_flag+=" $user_compiler_flags"
 
+case $GASNET_CONDUIT in
+  ibv|ofi|ucx) 
+    GASNET_RUNNER_ARG="${GASNET_RUNNER_ARG:-$GASNET_PREFIX/bin/gasnetrun_$GASNET_CONDUIT -n \${CAF_IMAGES:-2}}"
+  ;;
+  udp)
+    GASNET_RUNNER_ARG="${GASNET_RUNNER_ARG:-$GASNET_PREFIX/bin/amudprun -n \${CAF_IMAGES:-2}}"
+  ;;
+  mpi)
+    GASNET_RUNNER_ARG="${GASNET_RUNNER_ARG:-mpirun -n \${CAF_IMAGES:-2}}"
+  ;;
+  smp)
+    GASNET_RUNNER_ARG="${GASNET_RUNNER_ARG:-env GASNET_PSHM_NODES=\${CAF_IMAGES:-\${GASNET_PSHM_NODES:-}}}"
+  ;;
+  *)
+    GASNET_RUNNER_ARG="${GASNET_RUNNER_ARG:-}"
+  ;;
+esac
+
 RUN_FPM_SH="build/run-fpm.sh"
 cat << EOF > $RUN_FPM_SH
 #!/bin/sh
@@ -441,6 +487,10 @@ if echo "--help -help --version -version --list -list new update list clean publ
   set -x
   exec \$fpm "\$fpm_sub_cmd" "\$@"
 elif echo "build test run install" | grep -w -q -e "\$fpm_sub_cmd" ; then
+  sed -i.bak 's/^link = .*\$/$FPM_TOML_LINK_ENTRY/' build/fpm.toml
+  if test -n "$GASNET_RUNNER_ARG" && echo "test run" | grep -w -q -e "\$fpm_sub_cmd" ; then
+    set -- "--runner=$GASNET_RUNNER_ARG" "\$@"
+  fi
   set -x
   exec \$fpm "\$fpm_sub_cmd" \\
   --profile debug \\

src/caffeine/unit_test_parameters_m.F90

@@ -38,7 +38,7 @@ function subjob_setup() result(result_)
 
     call prif_sync_all()
     call prif_this_image_no_coarray(this_image=me)
-    result_ = (me == 1)
+    result_ = (me == 1) .and. (subjob_prefix /= "skip")
   end function