Browse Source

Initialise HDF5 MPI-IO performance tests

Peter Colberg 3 years ago
commit
b70ef94463

+ 19
- 0
LICENSE View File

@@ -0,0 +1,19 @@
1
+Copyright © 2015 Peter Colberg.
2
+
3
+Permission is hereby granted, free of charge, to any person obtaining a copy
4
+of this software and associated documentation files (the "Software"), to deal
5
+in the Software without restriction, including without limitation the rights
6
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+copies of the Software, and to permit persons to whom the Software is
8
+furnished to do so, subject to the following conditions:
9
+
10
+The above copyright notice and this permission notice shall be included in
11
+all copies or substantial portions of the Software.
12
+
13
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
16
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+THE SOFTWARE.

+ 39
- 0
README View File

@@ -0,0 +1,39 @@
1
+HDF5 MPI-IO performance tests
2
+=============================
3
+
4
+This repository provides a collection of self-contained programs that test
5
+the performance of the HDF5 MPI-IO driver. The purpose is to optimize the
6
+use of parallel HDF5 in the [nano-dimer] molecular simulation software.
7
+
8
+[nano-dimer]: http://colberg.org/nano-dimer/
9
+
10
+
11
+Usage
12
+-----
13
+
14
+The tests require an MPI-2 library and HDF5 1.8 with MPI-IO support.
15
+
16
+You may need to set `CPATH` and `LIBRARY_PATH` for compilation:
17
+
18
+~~~
19
+export CPATH="$HOME/usr/centos6-x86_64/hdf5-1.8.14/include"
20
+export LIBRARY_PATH="$HOME/usr/centos6-x86_64/hdf5-1.8.14/lib"
21
+~~~
22
+
23
+To compile all tests:
24
+
25
+~~~
26
+make -C test
27
+~~~
28
+
29
+To set hints for ROMIO, a commonly used MPI-IO implementation:
30
+
31
+~~~
32
+. examples/env.sh
33
+~~~
34
+
35
+To run a test on 32 processes with 2 processes per host:
36
+
37
+~~~
38
+mpirun -np 32 -npernode 2 hdf5_mpio_collective_write_double
39
+~~~

+ 2
- 0
examples/env.sh View File

@@ -0,0 +1,2 @@
1
+export ROMIO_HINTS="$PWD/romio-hints"
2
+export PATH="$PWD/test:$PATH"

+ 20
- 0
examples/romio-hints View File

@@ -0,0 +1,20 @@
1
+# This file sets hints to improve MPI-IO performance.
2
+#
3
+# To use this file, set the environment variable ROMIO_HINTS to the path.
4
+#
5
+# The hints supported by ROMIO are described in
6
+# http://www.mcs.anl.gov/research/projects/romio/doc/users-guide/node6.html
7
+#
8
+# The format of this file is described in
9
+# http://www.mcs.anl.gov/research/projects/romio/doc/users-guide/node13.html
10
+
11
+# Striping block size in bytes
12
+striping_unit 4194304
13
+# Collective buffering for collective write requests
14
+romio_cb_write enable
15
+# Collective buffering for collective read requests
16
+romio_cb_read enable
17
+# Data sieving for independent read requests
18
+romio_ds_read disable
19
+# Data sieving for independent write requests
20
+romio_ds_write disable

+ 9
- 0
test/Makefile View File

@@ -0,0 +1,9 @@
1
+CC = mpicc
2
+CFLAGS = -Wall -O2
3
+LIBS = -lhdf5
4
+LDFLAGS =
5
+
6
+all: $(patsubst %.c,%,$(wildcard *.c))
7
+
8
+%: %.c
9
+	$(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) $(LIBS)

+ 68
- 0
test/hdf5_mpio_collective_read_double.c View File

@@ -0,0 +1,68 @@
1
+/*
2
+ * Collectively read dataset using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+#include <stdlib.h>
11
+
12
+/*
13
+ * This program measures the time for collectively writing a dataset
14
+ * with fixed-size blocks of 256 MB per process. Each process reads
15
+ * from a contiguous hyperslab selection of the file dataspace. The
16
+ * offset of a selection is determined from the rank of a process.
17
+ *
18
+ * mpicc -Wall -O2 -o hdf5_mpio_collective_read_double hdf5_mpio_collective_read_double.c -lhdf5
19
+ */
20
+int main(int argc, char **argv)
21
+{
22
+  int rank, nranks;
23
+  hid_t file, fapl, file_space, mem_space, dset, dxpl;
24
+  double *buf;
25
+  double start, stop;
26
+
27
+  MPI_Init(&argc, &argv);
28
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
29
+  MPI_Comm_size(MPI_COMM_WORLD, &nranks);
30
+
31
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
32
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
33
+  file = H5Fopen("hdf5_mpio_collective_double.h5", H5F_ACC_RDONLY, fapl);
34
+  H5Pclose(fapl);
35
+
36
+  hsize_t mem_dims[1] = {33554432};
37
+  mem_space = H5Screate_simple(1, mem_dims, NULL);
38
+  buf = malloc(mem_dims[0]*sizeof(double));
39
+
40
+  dset = H5Dopen(file, "position", H5P_DEFAULT);
41
+  file_space = H5Dget_space(dset);
42
+
43
+  hsize_t offset[1] = {mem_dims[0]*rank};
44
+  H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, mem_dims, NULL);
45
+
46
+  dxpl = H5Pcreate(H5P_DATASET_XFER);
47
+  H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
48
+
49
+  MPI_Barrier(MPI_COMM_WORLD);
50
+  start = MPI_Wtime();
51
+  H5Dread(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, dxpl, buf);
52
+  H5Fflush(file, H5F_SCOPE_LOCAL);
53
+  MPI_Barrier(MPI_COMM_WORLD);
54
+  stop = MPI_Wtime();
55
+
56
+  hsize_t size = H5Dget_storage_size(dset);
57
+  if (rank == 0) printf("%g MB/s\n", size*0x1p-20/(stop-start));
58
+
59
+  H5Sclose(file_space);
60
+  H5Sclose(mem_space);
61
+  H5Pclose(dxpl);
62
+  H5Dclose(dset);
63
+  H5Fclose(file);
64
+  free(buf);
65
+
66
+  MPI_Finalize();
67
+  return 0;
68
+}

+ 78
- 0
test/hdf5_mpio_collective_read_double3.c View File

@@ -0,0 +1,78 @@
1
+/*
2
+ * Collectively read dataset using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+#include <stdlib.h>
11
+
12
+const hsize_t count[32] = {
13
+  11173874, 11172830, 11175068, 11180267, 11183949, 11182255, 11179574, 11173338,
14
+  11173973, 11170680, 11176392, 11175480, 11182974, 11175439, 11187152, 11180349,
15
+  11180009, 11174664, 11177009, 11186530, 11182741, 11191129, 11168457, 11166549,
16
+  11184881, 11170809, 11179326, 11189858, 11174405, 11190379, 11177826, 11175946
17
+};
18
+
19
+/*
20
+ * This program measures the time for collectively writing a dataset
21
+ * with variable-size blocks of ~256 MB per process, where the data
22
+ * consists of particles with 3-dimensional coordinates. Each process
23
+ * reads from a contiguous hyperslab selection of the file dataspace.
24
+ * The offset of a selection is determined using an exclusive scan
25
+ * operation over the number of particles per process.
26
+ *
27
+ * mpicc -Wall -O2 -o hdf5_mpio_collective_read_double3 hdf5_mpio_collective_read_double3.c -lhdf5
28
+ */
29
+int main(int argc, char **argv)
30
+{
31
+  int rank, nranks;
32
+  hid_t file, fapl, file_space, mem_space, dset, dxpl;
33
+  double *buf;
34
+  double start, stop;
35
+
36
+  MPI_Init(&argc, &argv);
37
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
38
+  MPI_Comm_size(MPI_COMM_WORLD, &nranks);
39
+
40
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
41
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
42
+  file = H5Fopen("hdf5_mpio_collective_double3.h5", H5F_ACC_RDONLY, fapl);
43
+  H5Pclose(fapl);
44
+
45
+  hsize_t mem_dims[2] = {count[rank%32], 3};
46
+  mem_space = H5Screate_simple(2, mem_dims, NULL);
47
+  buf = malloc(mem_dims[0]*mem_dims[1]*sizeof(double));
48
+
49
+  dset = H5Dopen(file, "position", H5P_DEFAULT);
50
+  file_space = H5Dget_space(dset);
51
+
52
+  hsize_t offset[2] = {0, 0};
53
+  MPI_Exscan(mem_dims, offset, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
54
+  H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, mem_dims, NULL);
55
+
56
+  dxpl = H5Pcreate(H5P_DATASET_XFER);
57
+  H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
58
+
59
+  MPI_Barrier(MPI_COMM_WORLD);
60
+  start = MPI_Wtime();
61
+  H5Dread(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, dxpl, buf);
62
+  H5Fflush(file, H5F_SCOPE_LOCAL);
63
+  MPI_Barrier(MPI_COMM_WORLD);
64
+  stop = MPI_Wtime();
65
+
66
+  hsize_t size = H5Dget_storage_size(dset);
67
+  if (rank == 0) printf("%g MB/s\n", size*0x1p-20/(stop-start));
68
+
69
+  H5Sclose(file_space);
70
+  H5Sclose(mem_space);
71
+  H5Pclose(dxpl);
72
+  H5Dclose(dset);
73
+  H5Fclose(file);
74
+  free(buf);
75
+
76
+  MPI_Finalize();
77
+  return 0;
78
+}

+ 88
- 0
test/hdf5_mpio_collective_read_double4.c View File

@@ -0,0 +1,88 @@
1
+/*
2
+ * Collectively read dataset using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+#include <stdlib.h>
11
+
12
+const hsize_t count[32] = {
13
+  11173874, 11172830, 11175068, 11180267, 11183949, 11182255, 11179574, 11173338,
14
+  11173973, 11170680, 11176392, 11175480, 11182974, 11175439, 11187152, 11180349,
15
+  11180009, 11174664, 11177009, 11186530, 11182741, 11191129, 11168457, 11166549,
16
+  11184881, 11170809, 11179326, 11189858, 11174405, 11190379, 11177826, 11175946
17
+};
18
+
19
+/*
20
+ * This program measures the time for collectively writing a dataset
21
+ * with variable-size blocks of ~256 MB per process, where the data
22
+ * consists of particles with 3-dimensional coordinates. Each process
23
+ * reads from a contiguous hyperslab selection of the file dataspace.
24
+ * The offset of a selection is determined using an exclusive scan
25
+ * operation over the number of particles per process.
26
+ *
27
+ * The particle coordinates are stored in memory as 4-component vectors,
28
+ * where only 3 components are actually used. This layout is typically
29
+ * found with parallel compute devices, e.g., GPUs, that require the memory
30
+ * alignment of vectors to be a power of two. The particle coordinates are
31
+ * written to a non-contiguous hyperslab selection of the memory dataspace.
32
+ *
33
+ * mpicc -Wall -O2 -o hdf5_mpio_collective_read_double4 hdf5_mpio_collective_read_double4.c -lhdf5
34
+ */
35
+int main(int argc, char **argv)
36
+{
37
+  int rank, nranks;
38
+  hid_t file, fapl, file_space, mem_space, dset, dxpl;
39
+  double *buf;
40
+  double start, stop;
41
+
42
+  MPI_Init(&argc, &argv);
43
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
44
+  MPI_Comm_size(MPI_COMM_WORLD, &nranks);
45
+
46
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
47
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
48
+  file = H5Fopen("hdf5_mpio_collective_double4.h5", H5F_ACC_RDONLY, fapl);
49
+  H5Pclose(fapl);
50
+
51
+  hsize_t mem_dims[2] = {count[rank%32], 4};
52
+  mem_space = H5Screate_simple(2, mem_dims, NULL);
53
+  buf = malloc(mem_dims[0]*mem_dims[1]*sizeof(double));
54
+
55
+  hsize_t mem_start[2] = {0, 0};
56
+  hsize_t mem_count[2] = {count[rank%32], 3};
57
+  H5Sselect_hyperslab(mem_space, H5S_SELECT_SET, mem_start, NULL, mem_count, NULL);
58
+
59
+  dset = H5Dopen(file, "position", H5P_DEFAULT);
60
+  file_space = H5Dget_space(dset);
61
+
62
+  hsize_t offset[2] = {0, 0};
63
+  MPI_Exscan(mem_dims, offset, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
64
+  H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, mem_count, NULL);
65
+
66
+  dxpl = H5Pcreate(H5P_DATASET_XFER);
67
+  H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
68
+
69
+  MPI_Barrier(MPI_COMM_WORLD);
70
+  start = MPI_Wtime();
71
+  H5Dread(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, dxpl, buf);
72
+  H5Fflush(file, H5F_SCOPE_LOCAL);
73
+  MPI_Barrier(MPI_COMM_WORLD);
74
+  stop = MPI_Wtime();
75
+
76
+  hsize_t size = H5Dget_storage_size(dset);
77
+  if (rank == 0) printf("%g MB/s\n", size*0x1p-20/(stop-start));
78
+
79
+  H5Sclose(file_space);
80
+  H5Sclose(mem_space);
81
+  H5Pclose(dxpl);
82
+  H5Dclose(dset);
83
+  H5Fclose(file);
84
+  free(buf);
85
+
86
+  MPI_Finalize();
87
+  return 0;
88
+}

+ 70
- 0
test/hdf5_mpio_collective_write_double.c View File

@@ -0,0 +1,70 @@
1
+/*
2
+ * Collectively write dataset using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+#include <stdlib.h>
11
+
12
+/*
13
+ * This program measures the time for collectively writing a dataset
14
+ * with fixed-size blocks of 256 MB per process. Each process writes
15
+ * to a contiguous hyperslab selection of the file dataspace. The
16
+ * offset of a selection is determined from the rank of a process.
17
+ *
18
+ * mpicc -Wall -O2 -o hdf5_mpio_collective_write_double hdf5_mpio_collective_write_double.c -lhdf5
19
+ */
20
+int main(int argc, char **argv)
21
+{
22
+  int rank, nranks;
23
+  hid_t file, fapl, file_space, mem_space, dset, dxpl;
24
+  double *buf;
25
+  double start, stop;
26
+
27
+  MPI_Init(&argc, &argv);
28
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
29
+  MPI_Comm_size(MPI_COMM_WORLD, &nranks);
30
+
31
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
32
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
33
+  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
34
+  file = H5Fcreate("hdf5_mpio_collective_double.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
35
+  H5Pclose(fapl);
36
+
37
+  hsize_t mem_dims[1] = {33554432};
38
+  mem_space = H5Screate_simple(1, mem_dims, NULL);
39
+  buf = malloc(mem_dims[0]*sizeof(double));
40
+
41
+  hsize_t file_dims[1] = {mem_dims[0]*nranks};
42
+  file_space = H5Screate_simple(1, file_dims, NULL);
43
+  dset = H5Dcreate(file, "position", H5T_NATIVE_DOUBLE, file_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
44
+
45
+  hsize_t offset[1] = {mem_dims[0]*rank};
46
+  H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, mem_dims, NULL);
47
+
48
+  dxpl = H5Pcreate(H5P_DATASET_XFER);
49
+  H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
50
+
51
+  MPI_Barrier(MPI_COMM_WORLD);
52
+  start = MPI_Wtime();
53
+  H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, dxpl, buf);
54
+  H5Fflush(file, H5F_SCOPE_LOCAL);
55
+  MPI_Barrier(MPI_COMM_WORLD);
56
+  stop = MPI_Wtime();
57
+
58
+  hsize_t size = H5Dget_storage_size(dset);
59
+  if (rank == 0) printf("%g MB/s\n", size*0x1p-20/(stop-start));
60
+
61
+  H5Sclose(file_space);
62
+  H5Sclose(mem_space);
63
+  H5Pclose(dxpl);
64
+  H5Dclose(dset);
65
+  H5Fclose(file);
66
+  free(buf);
67
+
68
+  MPI_Finalize();
69
+  return 0;
70
+}

+ 81
- 0
test/hdf5_mpio_collective_write_double3.c View File

@@ -0,0 +1,81 @@
1
+/*
2
+ * Collectively write dataset using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+#include <stdlib.h>
11
+
12
+const hsize_t count[32] = {
13
+  11173874, 11172830, 11175068, 11180267, 11183949, 11182255, 11179574, 11173338,
14
+  11173973, 11170680, 11176392, 11175480, 11182974, 11175439, 11187152, 11180349,
15
+  11180009, 11174664, 11177009, 11186530, 11182741, 11191129, 11168457, 11166549,
16
+  11184881, 11170809, 11179326, 11189858, 11174405, 11190379, 11177826, 11175946
17
+};
18
+
19
+/*
20
+ * This program measures the time for collectively writing a dataset
21
+ * with variable-size blocks of ~256 MB per process, where the data
22
+ * consists of particles with 3-dimensional coordinates. Each process
23
+ * writes to a contiguous hyperslab selection of the file dataspace.
24
+ * The offset of a selection is determined using an exclusive scan
25
+ * operation over the number of particles per process.
26
+ *
27
+ * mpicc -Wall -O2 -o hdf5_mpio_collective_write_double3 hdf5_mpio_collective_write_double3.c -lhdf5
28
+ */
29
+int main(int argc, char **argv)
30
+{
31
+  int rank, nranks;
32
+  hid_t file, fapl, file_space, mem_space, dset, dxpl;
33
+  double *buf;
34
+  double start, stop;
35
+
36
+  MPI_Init(&argc, &argv);
37
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
38
+  MPI_Comm_size(MPI_COMM_WORLD, &nranks);
39
+
40
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
41
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
42
+  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
43
+  file = H5Fcreate("hdf5_mpio_collective_double3.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
44
+  H5Pclose(fapl);
45
+
46
+  hsize_t mem_dims[2] = {count[rank%32], 3};
47
+  mem_space = H5Screate_simple(2, mem_dims, NULL);
48
+  buf = malloc(mem_dims[0]*mem_dims[1]*sizeof(double));
49
+
50
+  hsize_t file_dims[2] = {0, 3};
51
+  MPI_Allreduce(mem_dims, file_dims, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
52
+  file_space = H5Screate_simple(2, file_dims, NULL);
53
+  dset = H5Dcreate(file, "position", H5T_NATIVE_DOUBLE, file_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
54
+
55
+  hsize_t offset[2] = {0, 0};
56
+  MPI_Exscan(mem_dims, offset, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
57
+  H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, mem_dims, NULL);
58
+
59
+  dxpl = H5Pcreate(H5P_DATASET_XFER);
60
+  H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
61
+
62
+  MPI_Barrier(MPI_COMM_WORLD);
63
+  start = MPI_Wtime();
64
+  H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, dxpl, buf);
65
+  H5Fflush(file, H5F_SCOPE_LOCAL);
66
+  MPI_Barrier(MPI_COMM_WORLD);
67
+  stop = MPI_Wtime();
68
+
69
+  hsize_t size = H5Dget_storage_size(dset);
70
+  if (rank == 0) printf("%g MB/s\n", size*0x1p-20/(stop-start));
71
+
72
+  H5Sclose(file_space);
73
+  H5Sclose(mem_space);
74
+  H5Pclose(dxpl);
75
+  H5Dclose(dset);
76
+  H5Fclose(file);
77
+  free(buf);
78
+
79
+  MPI_Finalize();
80
+  return 0;
81
+}

+ 91
- 0
test/hdf5_mpio_collective_write_double4.c View File

@@ -0,0 +1,91 @@
1
+/*
2
+ * Collectively write dataset using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+#include <stdlib.h>
11
+
12
+const hsize_t count[32] = {
13
+  11173874, 11172830, 11175068, 11180267, 11183949, 11182255, 11179574, 11173338,
14
+  11173973, 11170680, 11176392, 11175480, 11182974, 11175439, 11187152, 11180349,
15
+  11180009, 11174664, 11177009, 11186530, 11182741, 11191129, 11168457, 11166549,
16
+  11184881, 11170809, 11179326, 11189858, 11174405, 11190379, 11177826, 11175946
17
+};
18
+
19
+/*
20
+ * This program measures the time for collectively writing a dataset
21
+ * with variable-size blocks of ~256 MB per process, where the data
22
+ * consists of particles with 3-dimensional coordinates. Each process
23
+ * writes to a contiguous hyperslab selection of the file dataspace.
24
+ * The offset of a selection is determined using an exclusive scan
25
+ * operation over the number of particles per process.
26
+ *
27
+ * The particle coordinates are stored in memory as 4-component vectors,
28
+ * where only 3 components are actually used. This layout is typically
29
+ * found with parallel compute devices, e.g., GPUs, that require the memory
30
+ * alignment of vectors to be a power of two. The particle coordinates are
31
+ * read from a non-contiguous hyperslab selection of the memory dataspace.
32
+ *
33
+ * mpicc -Wall -O2 -o hdf5_mpio_collective_write_double4 hdf5_mpio_collective_write_double4.c -lhdf5
34
+ */
35
+int main(int argc, char **argv)
36
+{
37
+  int rank, nranks;
38
+  hid_t file, fapl, file_space, mem_space, dset, dxpl;
39
+  double *buf;
40
+  double start, stop;
41
+
42
+  MPI_Init(&argc, &argv);
43
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
44
+  MPI_Comm_size(MPI_COMM_WORLD, &nranks);
45
+
46
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
47
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
48
+  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
49
+  file = H5Fcreate("hdf5_mpio_collective_double4.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
50
+  H5Pclose(fapl);
51
+
52
+  hsize_t mem_dims[2] = {count[rank%32], 4};
53
+  mem_space = H5Screate_simple(2, mem_dims, NULL);
54
+  buf = malloc(mem_dims[0]*mem_dims[1]*sizeof(double));
55
+
56
+  hsize_t mem_start[2] = {0, 0};
57
+  hsize_t mem_count[2] = {count[rank%32], 3};
58
+  H5Sselect_hyperslab(mem_space, H5S_SELECT_SET, mem_start, NULL, mem_count, NULL);
59
+
60
+  hsize_t file_dims[2] = {0, 3};
61
+  MPI_Allreduce(mem_dims, file_dims, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
62
+  file_space = H5Screate_simple(2, file_dims, NULL);
63
+  dset = H5Dcreate(file, "position", H5T_NATIVE_DOUBLE, file_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
64
+
65
+  hsize_t offset[2] = {0, 0};
66
+  MPI_Exscan(mem_dims, offset, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);
67
+  H5Sselect_hyperslab(file_space, H5S_SELECT_SET, offset, NULL, mem_count, NULL);
68
+
69
+  dxpl = H5Pcreate(H5P_DATASET_XFER);
70
+  H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE);
71
+
72
+  MPI_Barrier(MPI_COMM_WORLD);
73
+  start = MPI_Wtime();
74
+  H5Dwrite(dset, H5T_NATIVE_DOUBLE, mem_space, file_space, dxpl, buf);
75
+  H5Fflush(file, H5F_SCOPE_LOCAL);
76
+  MPI_Barrier(MPI_COMM_WORLD);
77
+  stop = MPI_Wtime();
78
+
79
+  hsize_t size = H5Dget_storage_size(dset);
80
+  if (rank == 0) printf("%g MB/s\n", size*0x1p-20/(stop-start));
81
+
82
+  H5Sclose(file_space);
83
+  H5Sclose(mem_space);
84
+  H5Pclose(dxpl);
85
+  H5Dclose(dset);
86
+  H5Fclose(file);
87
+  free(buf);
88
+
89
+  MPI_Finalize();
90
+  return 0;
91
+}

+ 60
- 0
test/hdf5_mpio_copy.c View File

@@ -0,0 +1,60 @@
1
+/*
2
+ * Copy contiguous dataset between files using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+
11
+/*
12
+ * This program measures the time for copying a dataset of 32M size
13
+ * between two parallel HDF5 files. The dataset has a contiguous layout,
14
+ * which means the dataset object consists of metadata and raw data.
15
+ *
16
+ * mpicc -Wall -O2 -o hdf5_mpio_copy hdf5_mpio_copy.c -lhdf5
17
+ */
18
+int main(int argc, char **argv)
19
+{
20
+  double start, stop;
21
+  int rank;
22
+  hid_t file1, file2, fapl, space, dset;
23
+
24
+  MPI_Init(&argc, &argv);
25
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
26
+
27
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
28
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
29
+  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
30
+
31
+  file1 = H5Fcreate("hdf5_mpio_copy_1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
32
+
33
+  hsize_t dims[1] = {4194304};
34
+  space = H5Screate_simple(1, dims, NULL);
35
+  dset = H5Dcreate(file1, "position", H5T_NATIVE_DOUBLE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
36
+  H5Sclose(space);
37
+  H5Dclose(dset);
38
+
39
+  H5Fflush(file1, H5F_SCOPE_LOCAL);
40
+  H5Fclose(file1);
41
+
42
+  file1 = H5Fopen("hdf5_mpio_copy_1.h5", H5F_ACC_RDONLY, fapl);
43
+  file2 = H5Fcreate("hdf5_mpio_copy_2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
44
+
45
+  MPI_Barrier(MPI_COMM_WORLD);
46
+  start = MPI_Wtime();
47
+  H5Ocopy(file1, "position", file2, "position", H5P_DEFAULT, H5P_DEFAULT);
48
+  H5Fflush(file2, H5F_SCOPE_LOCAL);
49
+  MPI_Barrier(MPI_COMM_WORLD);
50
+  stop = MPI_Wtime();
51
+
52
+  if (rank == 0) printf("%.3g s\n", stop-start);
53
+
54
+  H5Fclose(file1);
55
+  H5Fclose(file2);
56
+  H5Pclose(fapl);
57
+
58
+  MPI_Finalize();
59
+  return 0;
60
+}

+ 63
- 0
test/hdf5_mpio_copy_compact.c View File

@@ -0,0 +1,63 @@
1
+/*
2
+ * Copy compact dataset between files using HDF5 MPI-IO driver.
3
+ * Copyright © 2015 Peter Colberg.
4
+ * Distributed under the MIT license. (See accompanying file LICENSE.)
5
+ */
6
+
7
+#include <hdf5.h>
8
+#include <mpi.h>
9
+#include <stdio.h>
10
+
11
+/*
12
+ * This program measures the time for copying a dataset of 32K size
13
+ * between two parallel HDF5 files. The dataset has a compact layout,
14
+ * which means the dataset object consists of metadata only.
15
+ *
16
+ * mpicc -Wall -O2 -o hdf5_mpio_copy_compact hdf5_mpio_copy_compact.c -lhdf5
17
+ */
18
+int main(int argc, char **argv)
19
+{
20
+  double start, stop;
21
+  int rank;
22
+  hid_t file1, file2, fapl, space, dcpl, dset;
23
+
24
+  MPI_Init(&argc, &argv);
25
+  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
26
+
27
+  fapl = H5Pcreate(H5P_FILE_ACCESS);
28
+  H5Pset_fapl_mpio(fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
29
+  H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST);
30
+
31
+  file1 = H5Fcreate("hdf5_mpio_copy_compact_1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
32
+
33
+  hsize_t dims[1] = {4096};
34
+  space = H5Screate_simple(1, dims, NULL);
35
+  dcpl = H5Pcreate(H5P_DATASET_CREATE);
36
+  H5Pset_layout(dcpl, H5D_COMPACT);
37
+  dset = H5Dcreate(file1, "position", H5T_NATIVE_DOUBLE, space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
38
+  H5Sclose(space);
39
+  H5Pclose(dcpl);
40
+  H5Dclose(dset);
41
+
42
+  H5Fflush(file1, H5F_SCOPE_LOCAL);
43
+  H5Fclose(file1);
44
+
45
+  file1 = H5Fopen("hdf5_mpio_copy_compact_1.h5", H5F_ACC_RDONLY, fapl);
46
+  file2 = H5Fcreate("hdf5_mpio_copy_compact_2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
47
+
48
+  MPI_Barrier(MPI_COMM_WORLD);
49
+  start = MPI_Wtime();
50
+  H5Ocopy(file1, "position", file2, "position", H5P_DEFAULT, H5P_DEFAULT);
51
+  H5Fflush(file2, H5F_SCOPE_LOCAL);
52
+  MPI_Barrier(MPI_COMM_WORLD);
53
+  stop = MPI_Wtime();
54
+
55
+  if (rank == 0) printf("%.3g s\n", stop-start);
56
+
57
+  H5Fclose(file1);
58
+  H5Fclose(file2);
59
+  H5Pclose(fapl);
60
+
61
+  MPI_Finalize();
62
+  return 0;
63
+}

Loading…
Cancel
Save