Make the separate communication stream optional (elemental#184)

If this option is not set, the communication will land on the
Hydrogen-default stream. This _could_ be a separate stream from the
data compute stream, but in the LBANN use case, it generally will be
the compute stream.

Author: benson31

include/El/core/imports/aluminum.hpp

@@ -359,10 +359,25 @@ struct SyncInfoManager<Device::GPU>
 };
 #endif // HYDROGEN_HAVE_GPU
 
+inline bool use_separate_comm_stream() noexcept
+{
+    char const* const env = std::getenv("H_USE_SEPARATE_COMM_STREAM");
+    return (env && std::strlen(env) && env[0] != '0');
+}
+
 template <typename BackendT>
 SyncInfo<DeviceForBackend<BackendT>()> const& BackendSyncInfo()
 {
     constexpr Device D = DeviceForBackend<BackendT>();
+    if constexpr (D == El::Device::GPU)
+    {
+        static bool const use_separate_stream = use_separate_comm_stream();
+        if (!use_separate_stream)
+        {
+            return El::gpu::DefaultSyncInfo();
+        }
+    }
+
     static SyncInfoManager<D> si_mgr_(BackendT::Name());
     return si_mgr_.si_;
 }