dummy fi_join implementation with dummy mc handling

Signed-off-by: Tomasz Gromadzki <[email protected]>

Author: grom72

prov/sharp/src/sharp.h

@@ -72,7 +72,7 @@
 #define SHARP_IOV_LIMIT		1
 #define SHARP_TX_OP_FLAGS (0)
 #define SHARP_RX_OP_FLAGS (0)
-#define SHARP_DOMAIN_CAPS (FI_LOCAL_COMM | FI_REMOTE_COMM)
+#define SHARP_DOMAIN_CAPS (FI_COLLECTIVE | FI_LOCAL_COMM | FI_REMOTE_COMM)
 enum {
 	SHARP_RX_SIZE = 65536,
 	SHARP_TX_SIZE = 16384,
@@ -132,8 +132,9 @@ struct sharp_av {
 #define sharp_coll_comm_t void
 #endif
 struct sharp_mc {
-	struct util_coll_mc coll_mc;
-
+	struct fid_mc		mc_fid;
+	struct util_coll_mc coll_mc; //XXX will replace mc_fid in the future
+	struct fid_mc 		*oob_fid_mc;
 	struct sharp_ep		*ep;
 	sharp_coll_comm_t 	*sharp_context;
 };

prov/sharp/src/sharp_coll.c

@@ -43,8 +43,9 @@ int sharp_query_collective(struct fid_domain *domain,
 
 	switch (coll) {
 	case FI_BARRIER:
+		// return FI_SUCCESS; //XXX to be change when barier operation is implemented
 	case FI_ALLREDUCE:
-		return FI_SUCCESS;
+		// return FI_SUCCESS; //XXX to be change when allreduce operation is implemented
 	case FI_ALLGATHER:
 	case FI_SCATTER:
 	case FI_BROADCAST:
@@ -59,21 +60,43 @@ int sharp_query_collective(struct fid_domain *domain,
 	return -FI_ENOSYS;
 }
 
-#if 0
 static int sharp_mc_close(struct fid *fid)
 {
-#if 0
 	struct sharp_mc *mc;
 
 	mc = container_of(fid, struct sharp_mc, mc_fid.fid);
-	ofi_atomic_dec32(&mc->ep->ref);
+	if (mc->oob_fid_mc) {
+		fi_close(&(mc->oob_fid_mc->fid));
+		mc->oob_fid_mc = NULL;
+	}
+	// ofi_atomic_dec32(&mc->ep->ref); //XXX
 	free(mc);
 	return 0;
-#else
-	return -FI_ENOSYS;
-#endif
 }
 
+
+int	sharp_mc_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
+{
+	struct sharp_mc *mc;
+	struct fid_mc *fid_mc;
+	mc = container_of(fid, struct sharp_mc, mc_fid.fid);
+	fid_mc = container_of(bfid, struct fid_mc, fid);
+	mc->oob_fid_mc = fid_mc;
+	return 0;
+}
+
+
+static struct fi_ops sharp_mc_fid_ops = {
+	.size = sizeof(struct fi_ops),
+	.close = sharp_mc_close,
+	.bind = sharp_mc_bind,
+	.control = fi_no_control,
+	.ops_open = fi_no_ops_open,
+};
+
+#if 0
+
+
 #if 0
 static struct fi_ops sharp_mc_ops = {
 	.size = sizeof(struct fi_ops),
@@ -138,6 +161,16 @@ static int sharp_find_local_rank(struct fid_ep *ep, struct sharp_mc *sharp_mc)
 int sharp_join_collective(struct fid_ep *fid, const void *addr, uint64_t flags,
 		    struct fid_mc **mc_fid, void *context)
 {
+	struct sharp_mc *mc;
+	//XXX struct fi_peer_transfer_context *peer_context = context;
+
+	mc = calloc(1, sizeof(*mc));
+	if (!mc)
+		return -FI_ENOMEM;
+
+	*mc_fid = &mc->mc_fid;
+	(*mc_fid)->fid.ops = &sharp_mc_fid_ops;
+	return 0;
 
 #if 0
 	struct fi_collective_addr *c_addr;
@@ -180,8 +213,6 @@ int sharp_join_collective(struct fid_ep *fid, const void *addr, uint64_t flags,
 	*mc_fid = &new_mc->mc_fid;
 
 	return 0; //XXX
-#else
-	return -FI_ENOSYS;
 #endif
 }