Skip to content

Commit 94c9470

Browse files
wkliaowkliao
committed
WRF-IO add a benchmark option of moving data section when header grows
1 parent 47dbe91 commit 94c9470

File tree

2 files changed

+232
-10
lines changed

2 files changed

+232
-10
lines changed

‎benchmarks/WRF-IO/parallel_run.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ for i in ${check_PROGRAMS} ; do
8888
export PNETCDF_SAFE_MODE=$j
8989
# echo "PNETCDF_SAFE_MODE=$PNETCDF_SAFE_MODE PNETCDF_HINTS=$PNETCDF_HINTS"
9090

91-
CMD_OPTS="-q -d -y 100 -x 100 -i ${srcdir}/wrf_header.txt"
91+
CMD_OPTS="-q -d -g -n 2 -y 100 -x 100 -i ${srcdir}/wrf_header.txt"
9292
# echo "${LINENO}: ${MPIRUN} ./$i $CMD_OPTS -w $OUT_FILE.nc -r $OUT_FILE.nc"
9393
${MPIRUN} ./$i $CMD_OPTS -w $OUT_FILE.nc -r $OUT_FILE.nc
9494

‎benchmarks/WRF-IO/wrf_io.c‎

Lines changed: 231 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -845,7 +845,7 @@ int wrf_w_benchmark(char *out_file,
845845
char value[MPI_MAX_INFO_VAL+1];
846846
int flag;
847847

848-
printf("-----------------------------------------------------------\n");
848+
printf("\n-----------------------------------------------------------\n");
849849
printf("---- WRF-IO write benchmark ----\n");
850850
printf("Output NetCDF file name: %s\n", out_file);
851851
printf("Number of MPI processes: %d\n", nprocs);
@@ -915,8 +915,6 @@ int wrf_w_benchmark(char *out_file,
915915
}
916916
if (err != NC_NOERR) return err;
917917

918-
if (err != NC_NOERR) return err;
919-
920918
/* check if there is any PnetCDF internal malloc residue */
921919
MPI_Offset malloc_size, sum_size;
922920
err = ncmpi_inq_malloc_size(&malloc_size);
@@ -1100,7 +1098,7 @@ int wrf_r_benchmark(char *in_file,
11001098
char value[MPI_MAX_INFO_VAL+1];
11011099
int flag;
11021100

1103-
printf("-----------------------------------------------------------\n");
1101+
printf("\n-----------------------------------------------------------\n");
11041102
printf("---- WRF-IO read benchmark ----\n");
11051103
printf("Input NetCDF file name: %s\n", in_file);
11061104
printf("Number of MPI processes: %d\n", nprocs);
@@ -1164,6 +1162,218 @@ int wrf_r_benchmark(char *in_file,
11641162
}
11651163
if (err != NC_NOERR) return err;
11661164

1165+
/* check if there is any PnetCDF internal malloc residue */
1166+
MPI_Offset malloc_size, sum_size;
1167+
err = ncmpi_inq_malloc_size(&malloc_size);
1168+
if (err == NC_ENOTENABLED) /* --enable-profiling is not set at configure */
1169+
return NC_NOERR;
1170+
else if (err == NC_NOERR) {
1171+
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
1172+
if (rank == 0 && sum_size > 0)
1173+
printf("heap memory allocated by PnetCDF internally has %lld bytes yet to be freed\n",
1174+
sum_size);
1175+
if (malloc_size > 0) ncmpi_inq_malloc_list();
1176+
}
1177+
/* report the PnetCDF internal heap memory allocation high water mark */
1178+
err = ncmpi_inq_malloc_max_size(&malloc_size);
1179+
if (err == NC_NOERR) {
1180+
MPI_Reduce(&malloc_size, &sum_size, 1, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
1181+
if (verbose && rank == 0)
1182+
printf("Max heap memory allocated by PnetCDF internally is %.2f MiB\n\n",
1183+
(float)sum_size/1048576);
1184+
}
1185+
fflush(stdout);
1186+
1187+
return err;
1188+
}
1189+
1190+
static
1191+
int grow_header_benchmark(char *in_file)
1192+
{
1193+
char *attr;
1194+
int i, err=NC_NOERR, nprocs, rank, ncid, ndims, dimid[3];
1195+
int varid, unlimdimid, nvars, fix_nvars, rec_nvars;
1196+
double timing, max_t;
1197+
MPI_Offset hdr_size, hdr_extent, attr_len, num_rec, longitude, latitude;
1198+
MPI_Offset r_amnt[2], w_amnt[2], amnt[2], sum_amnt[2], fix_off, rec_off;
1199+
MPI_Offset rec_size;
1200+
1201+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1202+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
1203+
1204+
/* open input file */
1205+
err = ncmpi_open(MPI_COMM_WORLD, in_file, NC_WRITE, MPI_INFO_NULL, &ncid);
1206+
if (err != NC_NOERR) {
1207+
printf("Error at line=%d: opening file %s (%s)\n",
1208+
__LINE__, in_file, ncmpi_strerror(err));
1209+
goto err_out;
1210+
}
1211+
1212+
err = ncmpi_inq_dimid(ncid, "Time", &dimid[0]);
1213+
CHECK_ERR("ncmpi_inq_dimid")
1214+
err = ncmpi_inq_dimid(ncid, "south_north", &dimid[1]);
1215+
CHECK_ERR("ncmpi_inq_dimid")
1216+
err = ncmpi_inq_dimlen(ncid, dimid[1], &longitude);
1217+
CHECK_ERR("ncmpi_inq_dimlen")
1218+
err = ncmpi_inq_dimid(ncid, "west_east", &dimid[2]);
1219+
CHECK_ERR("ncmpi_inq_dimid")
1220+
err = ncmpi_inq_dimlen(ncid, dimid[2], &latitude);
1221+
CHECK_ERR("ncmpi_inq_dimlen")
1222+
1223+
err = ncmpi_inq_header_size(ncid, &hdr_size);
1224+
CHECK_ERR("ncmpi_inq_header_size")
1225+
err = ncmpi_inq_header_extent(ncid, &hdr_extent);
1226+
CHECK_ERR("ncmpi_inq_header_extent")
1227+
if (verbose && debug && rank == 0)
1228+
printf("Line %d: header size %lld extent %lld free space %lld\n",
1229+
__LINE__,hdr_size,hdr_extent,hdr_extent-hdr_size);
1230+
1231+
/* check number of records in input file */
1232+
err = ncmpi_inq_unlimdim(ncid, &unlimdimid);
1233+
CHECK_ERR("ncmpi_inq_unlimdim")
1234+
err = ncmpi_inq_dimlen(ncid, unlimdimid, &num_rec);
1235+
CHECK_ERR("ncmpi_inq_dimlen")
1236+
1237+
err = ncmpi_inq_nvars(ncid, &nvars);
1238+
CHECK_ERR("ncmpi_inq_nvars")
1239+
1240+
fix_nvars = 0;
1241+
rec_nvars = 0;
1242+
fix_off = -1;
1243+
rec_off = -1;
1244+
for (i=0; i<nvars; i++) {
1245+
err = ncmpi_inq_varndims(ncid, i, &ndims);
1246+
CHECK_ERR("ncmpi_inq_varndims");
1247+
if (ndims == 0) {
1248+
if (fix_off < 0) ncmpi_inq_varoffset(ncid, i, &fix_off);
1249+
fix_nvars++;
1250+
continue;
1251+
}
1252+
err = ncmpi_inq_vardimid(ncid, i, dimid);
1253+
CHECK_ERR("ncmpi_inq_vardimid");
1254+
if (dimid[0] == unlimdimid) {
1255+
rec_nvars++;
1256+
if (rec_off < 0) ncmpi_inq_varoffset(ncid, i, &rec_off);
1257+
}
1258+
else {
1259+
fix_nvars++;
1260+
if (fix_off < 0) ncmpi_inq_varoffset(ncid, i, &fix_off);
1261+
}
1262+
}
1263+
1264+
err = ncmpi_inq_recsize(ncid, &rec_size);
1265+
CHECK_ERR("ncmpi_inq_recsize")
1266+
1267+
if (verbose && debug && rank == 0) {
1268+
printf("Line %d: nvars %d fix_nvars %d rec_nvars %d num_rec %lld\n",
1269+
__LINE__,nvars,fix_nvars,rec_nvars, num_rec);
1270+
printf("Line %d: fix var offset %lld first record var offset %lld\n",
1271+
__LINE__,fix_off,rec_off);
1272+
}
1273+
1274+
/* re-enter define mode */
1275+
err = ncmpi_redef(ncid);
1276+
CHECK_ERR("ncmpi_redef")
1277+
1278+
err = ncmpi_def_var(ncid, "dummy_rec", NC_FLOAT, 3, dimid, &varid);
1279+
CHECK_ERR("ncmpi_def_var")
1280+
1281+
err = ncmpi_def_var_fill(ncid, varid, 0, NULL);
1282+
CHECK_ERR("ncmpi_def_var_fill")
1283+
1284+
/* add a global attribute to ensure header section grows */
1285+
attr_len = hdr_extent - hdr_size;
1286+
attr = (char*) malloc(attr_len);
1287+
for (i=0; i<attr_len; i++) attr[i] = 'a' + i % 26;
1288+
err = ncmpi_put_att_text(ncid, NC_GLOBAL, "dummy_attr", attr_len, attr);
1289+
CHECK_ERR("ncmpi_put_att_text")
1290+
free(attr);
1291+
1292+
err = ncmpi_inq_get_size(ncid, &r_amnt[0]);
1293+
CHECK_ERR("ncmpi_inq_get_size")
1294+
err = ncmpi_inq_put_size(ncid, &w_amnt[0]);
1295+
CHECK_ERR("ncmpi_inq_put_size")
1296+
1297+
err = 0;
1298+
if (verbose && debug && rank == 0)
1299+
printf("Line %d: rank %d r_amnt %lld w_amnt %lld\n",
1300+
__LINE__,rank,r_amnt[0],w_amnt[0]);
1301+
if (rank > 0 && r_amnt[0] > 0) {
1302+
printf("Line %d: rank %d r_amnt expect 0 but got %lld\n",
1303+
__LINE__,rank,r_amnt[0]);
1304+
err = 1;
1305+
}
1306+
if (rank > 0 && w_amnt[0] > 0) {
1307+
printf("Line %d: rank %d w_amnt expect 0 but got %lld\n",
1308+
__LINE__,rank,w_amnt[0]);
1309+
err = 1;
1310+
}
1311+
MPI_Allreduce(MPI_IN_PLACE, &err, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
1312+
if (err > 0) goto err_out;
1313+
1314+
/* start the timer */
1315+
MPI_Barrier(MPI_COMM_WORLD);
1316+
timing = MPI_Wtime();
1317+
1318+
err = ncmpi__enddef(ncid, 0, 0, 0, 0);
1319+
CHECK_ERR("ncmpi_enddef")
1320+
1321+
timing = MPI_Wtime() - timing;
1322+
1323+
err = ncmpi_inq_get_size(ncid, &r_amnt[1]);
1324+
CHECK_ERR("ncmpi_inq_get_size")
1325+
err = ncmpi_inq_put_size(ncid, &w_amnt[1]);
1326+
CHECK_ERR("ncmpi_inq_put_size")
1327+
1328+
err = ncmpi_inq_header_size(ncid, &hdr_size);
1329+
CHECK_ERR("ncmpi_inq_header_size")
1330+
err = ncmpi_inq_header_extent(ncid, &hdr_extent);
1331+
CHECK_ERR("ncmpi_inq_header_extent")
1332+
if (verbose && debug && rank == 0)
1333+
printf("Line %d: header size %lld extent %lld free space %lld\n",
1334+
__LINE__,hdr_size,hdr_extent,hdr_extent-hdr_size);
1335+
1336+
/* fill the new record variable, so ncmpidiff can run and check */
1337+
for (i=0; i<num_rec; i++) {
1338+
err = ncmpi_fill_var_rec(ncid, varid, i);
1339+
CHECK_ERR("ncmpi_fill_var_rec")
1340+
}
1341+
1342+
/* close file */
1343+
err = ncmpi_close(ncid);
1344+
CHECK_ERR("ncmpi_close")
1345+
1346+
/* process timing measurement */
1347+
amnt[0] = r_amnt[1] - r_amnt[0];
1348+
amnt[1] = w_amnt[1] - w_amnt[0];
1349+
MPI_Reduce(&timing, &max_t, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
1350+
MPI_Reduce(&amnt, &sum_amnt, 2, MPI_OFFSET, MPI_SUM, 0, MPI_COMM_WORLD);
1351+
if (verbose && rank == 0) {
1352+
double bw = (double)sum_amnt[1] / 1048576.0;
1353+
printf("\n-----------------------------------------------------------\n");
1354+
printf("---- WRF-IO header grow benchmark ----\n");
1355+
printf("Input NetCDF file name: %s\n", in_file);
1356+
printf("Number of MPI processes: %d\n", nprocs);
1357+
printf("Grid size longitude x latitude: %lld x %lld\n",longitude,latitude);
1358+
printf("Number of variables: %d\n", nvars);
1359+
printf("Number of fix-sized variables: %d\n", fix_nvars);
1360+
printf("Number of record variables: %d\n", rec_nvars);
1361+
printf("Number of time records: %lld\n",num_rec);
1362+
printf("Record size: %lld\n",rec_size);
1363+
printf(" %.2f MiB\n", (float)rec_size/1048576);
1364+
printf("Total read amount: %lld B\n", sum_amnt[0]);
1365+
printf(" %.2f MiB\n", (float)sum_amnt[0]/1048576);
1366+
printf(" %.2f GiB\n", (float)sum_amnt[0]/1073741824);
1367+
printf("Total write amount: %lld B\n", sum_amnt[1]);
1368+
printf(" %.2f MiB\n", (float)sum_amnt[1]/1048576);
1369+
printf(" %.2f GiB\n", (float)sum_amnt[1]/1073741824);
1370+
printf("Max time: %.4f sec\n", max_t);
1371+
printf("Write bandwidth: %.2f MiB/s\n", bw/max_t);
1372+
printf(" %.2f GiB/s\n", bw/1024.0/max_t);
1373+
printf("-----------------------------------------------------------\n");
1374+
}
1375+
1376+
err_out:
11671377
if (err != NC_NOERR) return err;
11681378

11691379
/* check if there is any PnetCDF internal malloc residue */
@@ -1241,6 +1451,7 @@ usage(char *argv0)
12411451
" [-h] print this help message\n"
12421452
" [-q] quiet mode (disable performance output)\n"
12431453
" [-d] debug mode\n"
1454+
" [-g] run header extent grow benchmark, requiring option -w\n"
12441455
" [-b] using PnetCDF blocking APIs (default: nonblocking)\n"
12451456
" [-r file1,file2,...] input files for read benchmark\n"
12461457
" [-w file1,file2,...] output files for write benchmark\n"
@@ -1255,9 +1466,9 @@ int main(int argc, char** argv)
12551466
{
12561467
extern int optind;
12571468
extern char *optarg;
1258-
char *out_files, *in_files, *cdl_file, **fname;
1259-
int i, err, nerrs=0, nprocs, rank, ntimes, psizes[2], hid;
1260-
int nfiles, do_read, do_write, blocking;
1469+
char *out_files, *in_files, *cdl_file, **fname, *hdr_grow_file=NULL;
1470+
int i, err, nprocs, rank, ntimes, psizes[2], hid;
1471+
int nfiles, do_read, do_write, blocking, do_hdr_grow;
12611472
MPI_Offset longitude, latitude;
12621473
MPI_Info info=MPI_INFO_NULL;
12631474

@@ -1277,8 +1488,9 @@ int main(int argc, char** argv)
12771488
out_files = NULL;
12781489
longitude = -1; /* default to use west_east from cdl file */
12791490
latitude = -1; /* default to use south_north from cdl file */
1491+
do_hdr_grow = 0;
12801492

1281-
while ((i = getopt(argc, argv, "hqdbr:w:y:x:n:c:i:")) != EOF)
1493+
while ((i = getopt(argc, argv, "hqdbgr:w:y:x:n:c:i:")) != EOF)
12821494
switch(i) {
12831495
case 'q': verbose = 0;
12841496
break;
@@ -1292,6 +1504,8 @@ int main(int argc, char** argv)
12921504
case 'w': do_write = 1;
12931505
out_files = strdup(optarg);
12941506
break;
1507+
case 'g': do_hdr_grow = 1;
1508+
break;
12951509
case 'y': longitude = atoll(optarg);
12961510
break;
12971511
case 'x': latitude = atoll(optarg);
@@ -1352,6 +1566,8 @@ int main(int argc, char** argv)
13521566
ntimes, blocking, info);
13531567

13541568
if (err != NC_NOERR) goto err_out;
1569+
if (do_hdr_grow == 1 && i == 0)
1570+
hdr_grow_file = strdup(fname[0]);
13551571
free(fname[i]);
13561572
}
13571573

@@ -1379,10 +1595,16 @@ int main(int argc, char** argv)
13791595
if (in_files != NULL) free(in_files);
13801596
}
13811597

1598+
if (do_write && do_hdr_grow && hdr_grow_file != NULL) {
1599+
err = grow_header_benchmark(hdr_grow_file);
1600+
if (err != NC_NOERR) goto err_out;
1601+
}
1602+
13821603
err_out:
13831604
if (info != MPI_INFO_NULL) MPI_Info_free(&info);
1605+
if (hdr_grow_file != NULL) free(hdr_grow_file);
13841606

13851607
MPI_Finalize();
1386-
return (nerrs > 0);
1608+
return (err != NC_NOERR);
13871609
}
13881610

0 commit comments

Comments
 (0)