Merge branch 'main' into treddy_issue_24_unpin_pytest

.github/workflows/main_ci.yml

@@ -14,33 +14,28 @@ jobs:
       matrix:
         platform: [ubuntu-latest,
                    macos-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     runs-on: ${{ matrix.platform }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
       # install this (logs) project
       - name: Install darshan_logs package
         run: |
           python -m pip install .
-      - name: obtain pydarshan pydarshan-devel branch
+      - name: obtain pydarshan
         run: |
           mkdir scratch
           cd scratch
           git clone https://github.com/darshan-hpc/darshan.git
-          cd darshan
-          git checkout pydarshan-devel
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade pytest
-          python -m pip install --upgrade lxml
-          # matplotlib is pinned because of
-          # gh-479
-          python -m pip install 'matplotlib<3.5'
+          python -m pip install --upgrade lxml matplotlib
       - if: ${{matrix.platform == 'macos-latest'}}
         name: Install MacOS deps
         run: |
@@ -53,7 +48,7 @@ jobs:
           git submodule update --init
           ./prepare.sh
           cd darshan-util
-          ./configure --prefix=$DARSHAN_INSTALL_PATH --enable-shared --enable-apxc-mod --enable-apmpi-mod
+          ./configure --prefix=$DARSHAN_INSTALL_PATH --enable-apxc-mod --enable-apmpi-mod
           make
           make install
       - name: Install pydarshan
@@ -70,6 +65,7 @@ jobs:
       - name: Run pytest to check for bad logs
         run: |
           export LD_LIBRARY_PATH=$PWD/scratch/darshan/darshan_install/lib
+          export DYLD_FALLBACK_LIBRARY_PATH=$PWD/scratch/darshan/darshan_install/lib
           # the test suite is sensitive to
           # relative dir for test file paths
           cd scratch/darshan/darshan-util/pydarshan

darshan_logs/hdf5_diagonal_write_only/README.md

@@ -0,0 +1,23 @@
+Set of example darshan logs that contain `H5D` and/or `H5F` module data
+to allow for "ground truth" verification for various data aggregation
+functions in the new Python summary report.
+
+The logs were generated using the Python `h5py` library with an HDF5
+configuration that was not using MPI-IO. The example application code
+used to generate these logs along with some of the execution details
+are available here: https://github.com/nawtrey/darshan_hdf5_testing
+
+A brief summary of each log's contents:
+
+| Log                                   | Modules                            | Description                                      |
+|---------------------------------------|------------------------------------|--------------------------------------------------|
+| `hdf5_file_opens_only`                | `POSIX`, `H5F`                     | 3 ranks open files                               |
+| `hdf5_diagonal_write_1_byte_dxt`      | `POSIX`, `DXT_POSIX`, `H5F`, `H5D` | 10 ranks open files, write 1 byte each           |
+| `hdf5_diagonal_write_bytes_range_dxt` | `POSIX`, `DXT_POSIX`, `H5F`, `H5D` | 10 ranks open files, write 10-100 bytes each     |
+| `hdf5_diagonal_write_half_flush_dxt`  | `POSIX`, `DXT_POSIX`, `H5F`, `H5D` | 10 ranks open files, write 1 byte, 5 ranks flush |
+| `hdf5_diagonal_write_half_ranks_dxt`  | `POSIX`, `DXT_POSIX`, `H5F`, `H5D` | 10 ranks open files, 5 ranks write 1 byte        |
+
+
+These were created on a single node of the LANL snow supercomputer. The
+commit hash of the darshan main branch used to build `darshan-runtime`
+is `c805d948`, and it was compiled with the `HEATMAP` module disabled.

darshan_logs/hdf5_orthogonality/README.md

@@ -0,0 +1,47 @@
+This log file contains H5D data, but not H5F data,
+to represent this new scenario that is now possible
+based on:
+https://github.com/darshan-hpc/darshan/pull/703
+
+The darshan runtime was compiled at main branch hash
+2c0ea4d5 with this configure command:
+
+./configure --prefix=/home/treddy/installs/darshan_install --with-log-path-by-env=DARSHAN_LOGPATH --with-jobid-env=SLURM_JOBID --enable-hdf5-mod --with-hdf5=/home/treddy/github_projects/spack/opt/spack/linux-ubuntu22.04-skylake/gcc-11.2.0/hdf5-1.12.2-epig3jbrnmusn47cj3ez7m5oto4gg3bk CC=mpicc
+
+The following short script and monitoring commands were used,
+along with the runtime configuration file at the bottom.
+
+
+```python
+import numpy as np
+from mpi4py import MPI
+import h5py
+
+
+def handle_io(rank_val, n_bytes):
+    bytes_to_write = np.void(bytes(1) * n_bytes)
+    with h5py.File(f"rank_{rank_val}.h5", "w") as f:
+        f.create_dataset("dataset", data=bytes_to_write)
+
+
+def main():
+    comm = MPI.COMM_WORLD
+    rank = comm.Get_rank()
+
+    handle_io(
+        rank_val=rank,
+        n_bytes=5,
+    )
+
+
+if __name__ == "__main__":
+    main()
+```
+
+mpirun -x LD_PRELOAD=/home/treddy/installs/darshan_install/lib/libdarshan.so:/home/treddy/github_projects/spack/opt/spack/linux-ubuntu22.04-skylake/gcc-11.2.0/hdf5-1.12.2-epig3jbrnmusn47cj3ez7m5oto4gg3bk/lib/libhdf5.so -x DARSHAN_CONFIG_PATH=/home/treddy/rough_work/darshan/issue_709/runtime_config.txt -x DARSHAN_LOGPATH=/home/treddy/rough_work/darshan/issue_709 -np 3 python test.py
+
+```
+MOD_ENABLE STDIO,POSIX,DXT_POSIX,H5D
+MOD_DISABLE H5F
+DISABLE_SHARED_REDUCTION 1
+```

darshan_logs/ior_pnetcdf_hdf5/README.md

@@ -0,0 +1,8 @@
+This directory contains 2 Darshan logs corresponding to 2 identical runs of IOR:
+ - 1 using PnetCDF backend
+ - 1 using HDF5 backend
+
+Each run uses 4 processes and default IOR parameters. The Darshan logs contain
+instrumentation for the high-level I/O library (i.e., PnetCDF or HDF5) used in
+each case. These logs are used as part of a CI test to ensure similarity of
+Darshan output for each high-level I/O library.

darshan_logs/release_logs/README.md

@@ -0,0 +1,11 @@
+This directory contains logs generated for every 3.x version of Darshan.
+These logs are generated as part of the release process and are intended
+to ensure Darshan maintains backwards compatibility for parsing log files
+generated by older versions of the library.
+
+These test logs are generated with the mpi-io-test benchmark that is
+distributed as part of Darshan's internal regression testing framework
+(darshan-test/regression/test-cases/src/mpi-io-test.c). The benchmark
+is executed using 4 processes and uses all default parameters to mpi-io-test.
+Logs are generated on an x86_64 architecture for all releases. Logs are
+also generated on ppc architecure up until version 3.1.8.

darshan_logs/runtime_heatmap_inactive_ranks/README.md

@@ -0,0 +1,37 @@
+This is an example darshan log that contains both runtime
+`HEATMAP` and `STDIO` module data, to facilitate reproduction
+of an issue with the runtime heatmap data handling at the pydarshan level
+when a subset of ranks are inactive. There should be 20 active
+and 20 inactive ranks at the `STDIO` level.
+
+The darshan runtime was built off `main` branch at
+hash `548d9aed` on a LANL `darwin` `skylake-gold` node
+
+```
+module load gcc openmpi
+./prepare.sh
+cd darshan-runtime/
+./configure --prefix=/vast/home/treddy/darshan_install --with-log-path-by-env=DARSHAN_LOGPATH --with-jobid-env=SLURM_JOBID CC=mpicc
+make
+make install
+```
+
+The C code, compiled with mpicc, and monitoring incantation are below:
+
+```
+#include <mpi.h>
+#include <stdio.h>
+
+int main(int argc, char** argv) {
+    MPI_Init(NULL, NULL);
+    int world_rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    if (world_rank % 2 == 0) {
+        printf("Hello world from rank %i\n", world_rank);
+    }
+    MPI_Finalize();
+    return 0;
+}
+```
+
+mpirun -x DXT_ENABLE_IO_TRACE=1 -x DARSHAN_LOGPATH=/vast/home/treddy/rough_work/darshan/issue_730 -x DARSHAN_DISABLE_SHARED_REDUCTION=1 -x LD_PRELOAD=/vast/home/treddy/darshan_install/lib/libdarshan.so -np 40 a.out

pyproject.toml

@@ -6,8 +6,8 @@ requires = [
 
 [project]
 name = "darshan_logs"
-
-requires-python = ">=3.7"
+dynamic = ["version"]
+requires-python = ">=3.6"
 
 [project.urls]
 source = "https://github.com/darshan-hpc/darshan-logs"