From 8ad370a7843549dd2c53176c14dbbe803660e2c9 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 11 Nov 2022 21:14:21 -0700 Subject: [PATCH 01/42] fix for issue #1209 --- include/netCDF4.pxi | 8 +++----- setup.py | 14 ++++++++++++-- src/netCDF4/_netCDF4.pyx | 20 +++++++++++++------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index 86e309839..864436a0d 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -368,6 +368,7 @@ cdef extern from "netcdf.h": int nc_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier) nogil + IF HAS_QUANTIZATION_SUPPORT: cdef extern from "netcdf.h": cdef enum: @@ -377,6 +378,8 @@ IF HAS_QUANTIZATION_SUPPORT: NC_QUANTIZE_BITROUND int nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd) nogil int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil + +IF HAS_NCFILTER: cdef extern from "netcdf_filter.h": int nc_inq_filter_avail(int ncid, unsigned filterid) nogil @@ -388,8 +391,6 @@ IF HAS_SZIP_SUPPORT: int nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp) nogil int nc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_bloc) nogil int nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp) nogil - cdef extern from "netcdf_filter.h": - int nc_inq_filter_avail(int ncid, unsigned filterid) nogil IF HAS_ZSTANDARD_SUPPORT: cdef extern from "netcdf_filter.h": @@ -397,7 +398,6 @@ IF HAS_ZSTANDARD_SUPPORT: H5Z_FILTER_ZSTD int nc_def_var_zstandard(int ncid, int varid, int level) nogil int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) nogil - int nc_inq_filter_avail(int ncid, unsigned id) nogil IF HAS_BZIP2_SUPPORT: cdef extern from "netcdf_filter.h": @@ -405,7 +405,6 @@ IF HAS_BZIP2_SUPPORT: H5Z_FILTER_BZIP2 int nc_def_var_bzip2(int ncid, int varid, int level) nogil int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp) nogil - int nc_inq_filter_avail(int ncid, unsigned filterid) nogil IF HAS_BLOSC_SUPPORT: cdef extern from "netcdf_filter.h": @@ -413,7 +412,6 @@ IF HAS_BLOSC_SUPPORT: H5Z_FILTER_BLOSC int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle) nogil int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep) nogil - int nc_inq_filter_avail(int ncid, unsigned filterid) nogil IF HAS_NC_OPEN_MEM: cdef extern from "netcdf_mem.h": diff --git a/setup.py b/setup.py index 87ba082e9..b1e941a96 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ def check_api(inc_dirs,netcdf_lib_version): has_zstandard = False has_bzip2 = False has_blosc = False + has_ncfilter = False has_set_alignment = False for d in inc_dirs: @@ -116,6 +117,8 @@ def check_api(inc_dirs,netcdf_lib_version): has_bzip2 = True if line.startswith('EXTERNL int nc_def_var_blosc'): has_blosc = True + if line.startswith('EXTERNL int nc_inq_filter_avail'): + has_ncfilter = True ncmetapath = os.path.join(d,'netcdf_meta.h') if os.path.exists(ncmetapath): @@ -143,7 +146,7 @@ def check_api(inc_dirs,netcdf_lib_version): return has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \ has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ - has_zstandard, has_bzip2, has_blosc, has_set_alignment + has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter def getnetcdfvers(libdirs): @@ -557,7 +560,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_rename_grp, has_nc_inq_path, has_nc_inq_format_extended, \ has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ - has_zstandard, has_bzip2, has_blosc, has_set_alignment = \ + has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter = \ check_api(inc_dirs,netcdf_lib_version) # for netcdf 4.4.x CDF5 format is always enabled. if netcdf_lib_version is not None and\ @@ -671,6 +674,13 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): sys.stdout.write('netcdf lib does not have nc_set_alignment function\n') f.write('DEF HAS_SET_ALIGNMENT = 0\n') + if has_blosc: + sys.stdout.write('netcdf lib has nc_inq_filter_avail function\n') + f.write('DEF HAS_NCFILTER = 1\n') + else: + sys.stdout.write('netcdf lib does not have nc_inq_filter_avail function\n') + f.write('DEF HAS_NCFILTER = 0\n') + f.close() if has_parallel4_support or has_pnetcdf_support: diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index 29852a65d..aa64adde1 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -3543,15 +3543,21 @@ returns True if bzip2 compression filter is available""" **`has_szip_filter(self)`** returns True if szip compression filter is available""" cdef int ierr - IF HAS_SZIP_SUPPORT: - with nogil: - ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_SZIP) - if ierr: + IF HAS_NCFILTER: + IF HAS_SZIP_SUPPORT: + with nogil: + ierr = nc_inq_filter_avail(self._grpid, H5Z_FILTER_SZIP) + if ierr: + return False + else: + return True + ELSE: return False - else: + ELSE: + IF HAS_SZIP_SUPPORT: return True - ELSE: - return False + ELSE: + return False cdef class Group(Dataset): """ From b67c0f7c17a170f3408bc25c76a4d625b276eb16 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 11 Nov 2022 21:15:34 -0700 Subject: [PATCH 02/42] update --- Changelog | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index 41baa59c7..dbbbed650 100644 --- a/Changelog +++ b/Changelog @@ -1,10 +1,11 @@ - version 1.6.2 (Unrelease) -========================== + version 1.6.2 (not yet released) +============================== * Added ``netCDF4.__has_set_alignment__`` property to help identify if the underlying netcdf4 supports setting the HDF5 alignment. * Slicing multi-dimensional variables with an all False boolean index array now returns an empty numpy array (instead of raising an exception - issue #1197). Behavior now consistent with numpy slicing. + * fix problem with compiling using netcdf-c < 4.9.0 (issue #1209) version 1.6.1 (tag v1.6.1rel) ============================== From b088979a57627fcc1e4d2117fa5291d8d698eb31 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 11 Nov 2022 21:20:57 -0700 Subject: [PATCH 03/42] add new test with older netcdf-c (4.7.4) --- .../workflows/{build.yml => build_latest.yml} | 2 +- .github/workflows/build_old.yml | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) rename .github/workflows/{build.yml => build_latest.yml} (98%) create mode 100644 .github/workflows/build_old.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build_latest.yml similarity index 98% rename from .github/workflows/build.yml rename to .github/workflows/build_latest.yml index 11e218157..ddbfd7680 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build_latest.yml @@ -1,4 +1,4 @@ -name: Build and Test Linux +name: Build and Test Linux with latest netcdf-c on: [push, pull_request] jobs: build-linux: diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml new file mode 100644 index 000000000..c17678b66 --- /dev/null +++ b/.github/workflows/build_old.yml @@ -0,0 +1,104 @@ +name: Build and Test Linux with older netcdf-c +on: [push, pull_request] +jobs: + build-linux: + name: Python (${{ matrix.python-version }}) + runs-on: ubuntu-latest + env: + PNETCDF_VERSION: 1.12.1 + NETCDF_VERSION: 4.7.4 + NETCDF_DIR: ${{ github.workspace }}/.. + NETCDF_EXTRA_CONFIG: --enable-pnetcdf + CC: mpicc.mpich + #NO_NET: 1 + strategy: + matrix: + python-version: ["3.9"] + steps: + + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Ubuntu Dependencies + run: | + sudo apt-get update + sudo apt-get install mpich libmpich-dev libhdf5-mpich-dev libcurl4-openssl-dev bzip2 libsnappy-dev libblosc-dev libzstd-dev + echo "Download and build PnetCDF version ${PNETCDF_VERSION}" + wget https://parallel-netcdf.github.io/Release/pnetcdf-${PNETCDF_VERSION}.tar.gz + tar -xzf pnetcdf-${PNETCDF_VERSION}.tar.gz + pushd pnetcdf-${PNETCDF_VERSION} + ./configure --prefix $NETCDF_DIR --enable-shared --disable-fortran --disable-cxx + make -j 2 + make install + popd + echo "Download and build netCDF version ${NETCDF_VERSION}" + wget https://downloads.unidata.ucar.edu/netcdf-c/4.9.0/netcdf-c-${NETCDF_VERSION}.tar.gz + tar -xzf netcdf-c-${NETCDF_VERSION}.tar.gz + pushd netcdf-c-${NETCDF_VERSION} + export CPPFLAGS="-I/usr/include/hdf5/mpich -I${NETCDF_DIR}/include" + export LDFLAGS="-L${NETCDF_DIR}/lib" + export LIBS="-lhdf5_mpich_hl -lhdf5_mpich -lm -lz" + ./configure --prefix $NETCDF_DIR --enable-netcdf-4 --enable-shared --enable-dap --enable-parallel4 $NETCDF_EXTRA_CONFIG + make -j 2 + make install + popd + +# - name: The job has failed +# if: ${{ failure() }} +# run: | +# cd netcdf-c-${NETCDF_VERSION} +# cat config.log + + - name: Install python dependencies via pip + run: | + python -m pip install --upgrade pip + pip install numpy cython cftime pytest twine wheel check-manifest mpi4py + + - name: Install netcdf4-python + run: | + export PATH=${NETCDF_DIR}/bin:${PATH} + export NETCDF_PLUGIN_DIR=${{ github.workspace }}/netcdf-c-${NETCDF_VERSION}/plugins/plugindir + python setup.py install + - name: Test + run: | + export PATH=${NETCDF_DIR}/bin:${PATH} + python checkversion.py + # serial + cd test + python run_all.py + # parallel (hdf5 for netcdf4, pnetcdf for netcdf3) + cd ../examples + mpirun.mpich -np 4 python mpi_example.py + if [ $? -ne 0 ] ; then + echo "hdf5 mpi test failed!" + exit 1 + else + echo "hdf5 mpi test passed!" + fi + mpirun.mpich -np 4 python mpi_example_compressed.py + if [ $? -ne 0 ] ; then + echo "hdf5 compressed mpi test failed!" + exit 1 + else + echo "hdf5 compressed mpi test passed!" + fi + mpirun.mpich -np 4 python mpi_example.py NETCDF3_64BIT_DATA + if [ $? -ne 0 ] ; then + echo "pnetcdf mpi test failed!" + exit 1 + else + echo "pnetcdf mpi test passed!" + fi + + - name: Tarball + run: | + export PATH=${NETCDF_DIR}/bin:${PATH} + python setup.py --version + check-manifest --version + check-manifest --verbose + pip wheel . -w dist --no-deps + twine check dist/* From 9ade9ad07fbbbe397824a6ebc64a5db65fa01bf2 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 11 Nov 2022 21:22:50 -0700 Subject: [PATCH 04/42] fix typo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1e941a96..3de06408b 100644 --- a/setup.py +++ b/setup.py @@ -674,7 +674,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): sys.stdout.write('netcdf lib does not have nc_set_alignment function\n') f.write('DEF HAS_SET_ALIGNMENT = 0\n') - if has_blosc: + if has_ncfilter: sys.stdout.write('netcdf lib has nc_inq_filter_avail function\n') f.write('DEF HAS_NCFILTER = 1\n') else: From 6db3ce7ad6f03a68b07e8c39b159cf1bdfa515bc Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 11 Nov 2022 21:24:25 -0700 Subject: [PATCH 05/42] fix formatting --- src/netCDF4/_netCDF4.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index aa64adde1..605d3957e 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -3553,11 +3553,11 @@ returns True if szip compression filter is available""" return True ELSE: return False - ELSE: - IF HAS_SZIP_SUPPORT: - return True - ELSE: - return False + ELSE: + IF HAS_SZIP_SUPPORT: + return True + ELSE: + return False cdef class Group(Dataset): """ From 21f49f546492412b234a18cc794fadd8d57e3326 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 11 Nov 2022 21:25:18 -0700 Subject: [PATCH 06/42] remove whitespace --- include/netCDF4.pxi | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/netCDF4.pxi b/include/netCDF4.pxi index 864436a0d..9233c8165 100644 --- a/include/netCDF4.pxi +++ b/include/netCDF4.pxi @@ -367,8 +367,6 @@ cdef extern from "netcdf.h": int nc_inq_enum_ident(int ncid, nc_type xtype, long long value, char *identifier) nogil - - IF HAS_QUANTIZATION_SUPPORT: cdef extern from "netcdf.h": cdef enum: From c6fee378fb77216e7b94b25f1df89f1529dfd064 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 10:28:49 -0700 Subject: [PATCH 07/42] fix download url --- .github/workflows/build_latest.yml | 4 ++-- .github/workflows/build_old.yml | 6 +++--- .github/workflows/miniconda.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index ddbfd7680..10e7afd6a 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -13,7 +13,7 @@ jobs: #NO_NET: 1 strategy: matrix: - python-version: ["3.9"] + python-version: ["3.10"] steps: - uses: actions/checkout@v2 @@ -36,7 +36,7 @@ jobs: make install popd echo "Download and build netCDF version ${NETCDF_VERSION}" - wget https://downloads.unidata.ucar.edu/netcdf-c/4.9.0/netcdf-c-${NETCDF_VERSION}.tar.gz + wget https://downloads.unidata.ucar.edu/netcdf-c/${NETCDF_VERSION}/netcdf-c-${NETCDF_VERSION}.tar.gz tar -xzf netcdf-c-${NETCDF_VERSION}.tar.gz pushd netcdf-c-${NETCDF_VERSION} export CPPFLAGS="-I/usr/include/hdf5/mpich -I${NETCDF_DIR}/include" diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index c17678b66..c4fe0ad29 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -6,14 +6,14 @@ jobs: runs-on: ubuntu-latest env: PNETCDF_VERSION: 1.12.1 - NETCDF_VERSION: 4.7.4 + NETCDF_VERSION: 4.8.1 NETCDF_DIR: ${{ github.workspace }}/.. NETCDF_EXTRA_CONFIG: --enable-pnetcdf CC: mpicc.mpich #NO_NET: 1 strategy: matrix: - python-version: ["3.9"] + python-version: ["3.10"] steps: - uses: actions/checkout@v2 @@ -36,7 +36,7 @@ jobs: make install popd echo "Download and build netCDF version ${NETCDF_VERSION}" - wget https://downloads.unidata.ucar.edu/netcdf-c/4.9.0/netcdf-c-${NETCDF_VERSION}.tar.gz + wget https://downloads.unidata.ucar.edu/netcdf-c/${NETCDF_VERSION}/netcdf-c-${NETCDF_VERSION}.tar.gz tar -xzf netcdf-c-${NETCDF_VERSION}.tar.gz pushd netcdf-c-${NETCDF_VERSION} export CPPFLAGS="-I/usr/include/hdf5/mpich -I${NETCDF_DIR}/include" diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 601fbda04..6e132887e 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -53,7 +53,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [ "3.9" ] + python-version: [ "3.10" ] os: [ubuntu-latest] platform: [x64] steps: From 87667e8b24b209ea2f6970dde267d93178b87ce1 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 10:38:05 -0700 Subject: [PATCH 08/42] add 'oversubscribe' option to mpirun --- .github/workflows/miniconda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 6e132887e..8c1c3427d 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -88,8 +88,8 @@ jobs: export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" which mpirun mpirun --version - #mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi - mpirun -np 4 python mpi_example.py + mpirun -np 4 --oversubscribe python mpi_example.py # for openmpi + #mpirun -np 4 python mpi_example.py if [ $? -ne 0 ] ; then echo "hdf5 mpi test failed!" exit 1 From e544923fbd9f8e6e5f1690168c3478c1d629ed5c Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 10:44:47 -0700 Subject: [PATCH 09/42] bump version number --- .github/workflows/build_master.yml | 2 +- .github/workflows/miniconda.yml | 2 +- src/netCDF4/_netCDF4.pyx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_master.yml b/.github/workflows/build_master.yml index 00b3a52d4..1749d9585 100644 --- a/.github/workflows/build_master.yml +++ b/.github/workflows/build_master.yml @@ -10,7 +10,7 @@ jobs: #NO_NET: 1 strategy: matrix: - python-version: ["3.9"] + python-version: ["3.10"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 8c1c3427d..f8168ac3d 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -12,7 +12,7 @@ jobs: # NO_NET: 1 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10" ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] os: [windows-latest, ubuntu-latest, macos-latest] platform: [x64, x32] exclude: diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index 605d3957e..c36f308b1 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1,5 +1,5 @@ """ -Version 1.6.1 +Version 1.6.2 ------------- # Introduction @@ -1230,7 +1230,7 @@ if sys.version_info[0:2] < (3, 7): # Python 3.7+ guarantees order; older versions need OrderedDict from collections import OrderedDict -__version__ = "1.6.1" +__version__ = "1.6.2" # Initialize numpy import posixpath From 1edc33143625c2be6091e0601a3d7f600655a45e Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 10:52:45 -0700 Subject: [PATCH 10/42] add -v to pip install --- .github/workflows/miniconda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index f8168ac3d..94258a883 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -34,7 +34,7 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - pip install -e . --no-deps --force-reinstall + pip install -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From 6176b3b01a2b7708c31f93120f55c3ae9017bdd1 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 10:58:54 -0700 Subject: [PATCH 11/42] try again to get verbose pip output --- .github/workflows/miniconda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 94258a883..98487b460 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -34,7 +34,7 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - pip install -v -e . --no-deps --force-reinstall + pip install -e -v -v -v . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} @@ -70,7 +70,7 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - pip install -e . --no-deps --force-reinstall + pip install -e -v -v -v . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From 3a9cd068fcdd6395b27a3f493d1504220bd6fc30 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 11:01:19 -0700 Subject: [PATCH 12/42] try again --- .github/workflows/miniconda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 98487b460..e6fe361af 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -34,7 +34,7 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - pip install -e -v -v -v . --no-deps --force-reinstall + pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} @@ -70,7 +70,7 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - pip install -e -v -v -v . --no-deps --force-reinstall + pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From df65aa24319b19b92f1da83cc1809d76e85d7c54 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 11:07:17 -0700 Subject: [PATCH 13/42] get nc-config output --- .github/workflows/miniconda.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index e6fe361af..f82deb57c 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -70,6 +70,8 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config + which nc-config + nc-config --all pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda From a751198a50785e71ad7e393c20556391fbd0e8b9 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 11:18:33 -0700 Subject: [PATCH 14/42] add debug print to diagnose mpi failure --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 3de06408b..079525caa 100644 --- a/setup.py +++ b/setup.py @@ -121,6 +121,7 @@ def check_api(inc_dirs,netcdf_lib_version): has_ncfilter = True ncmetapath = os.path.join(d,'netcdf_meta.h') + print('ncmetapath = %s' % ncmetapath) if os.path.exists(ncmetapath): for line in open(ncmetapath): if line.startswith('#define NC_HAS_CDF5'): @@ -128,6 +129,7 @@ def check_api(inc_dirs,netcdf_lib_version): if line.startswith('#define NC_HAS_PARALLEL'): has_parallel_support = bool(int(line.split()[2])) if line.startswith('#define NC_HAS_PARALLEL4'): + print('nc_has_paralle4',line) has_parallel4_support = bool(int(line.split()[2])) if line.startswith('#define NC_HAS_PNETCDF'): has_pnetcdf_support = bool(int(line.split()[2])) From 8389895a690494d72cb389ff7eca02a24800aff3 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 11:28:53 -0700 Subject: [PATCH 15/42] add more debug prints --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 079525caa..33df616f1 100644 --- a/setup.py +++ b/setup.py @@ -131,6 +131,7 @@ def check_api(inc_dirs,netcdf_lib_version): if line.startswith('#define NC_HAS_PARALLEL4'): print('nc_has_paralle4',line) has_parallel4_support = bool(int(line.split()[2])) + print('has_parallel4_support',has_parallel4_support) if line.startswith('#define NC_HAS_PNETCDF'): has_pnetcdf_support = bool(int(line.split()[2])) if line.startswith('#define NC_HAS_SZIP_WRITE'): @@ -563,6 +564,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter = \ + check_api(inc_dirs,netcdf_lib_version) # for netcdf 4.4.x CDF5 format is always enabled. if netcdf_lib_version is not None and\ @@ -573,6 +575,8 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): try: import mpi4py except ImportError: + if has_parallel4_support or has_pnetcdf_support: + print('no mpi4py found, disabling mpi parallel support') has_parallel4_support = False has_pnetcdf_support = False From 6ad7e43ffb47c22933ee5fa6626a4eaa722562e4 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 11:33:55 -0700 Subject: [PATCH 16/42] update --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 33df616f1..468e35017 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,6 @@ def check_api(inc_dirs,netcdf_lib_version): has_ncfilter = True ncmetapath = os.path.join(d,'netcdf_meta.h') - print('ncmetapath = %s' % ncmetapath) if os.path.exists(ncmetapath): for line in open(ncmetapath): if line.startswith('#define NC_HAS_CDF5'): @@ -129,7 +128,6 @@ def check_api(inc_dirs,netcdf_lib_version): if line.startswith('#define NC_HAS_PARALLEL'): has_parallel_support = bool(int(line.split()[2])) if line.startswith('#define NC_HAS_PARALLEL4'): - print('nc_has_paralle4',line) has_parallel4_support = bool(int(line.split()[2])) print('has_parallel4_support',has_parallel4_support) if line.startswith('#define NC_HAS_PNETCDF'): @@ -576,7 +574,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): import mpi4py except ImportError: if has_parallel4_support or has_pnetcdf_support: - print('no mpi4py found, disabling mpi parallel support') + f.write('no mpi4py found, disabling mpi parallel support\n') has_parallel4_support = False has_pnetcdf_support = False From 7cd8ee831722be03a5bb01c7cf445d6e93c2362d Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 11:54:06 -0700 Subject: [PATCH 17/42] fix typo --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 468e35017..1fdd37f88 100644 --- a/setup.py +++ b/setup.py @@ -562,7 +562,6 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format, has_nc_open_mem, has_nc_create_mem, \ has_parallel4_support, has_pnetcdf_support, has_szip_support, has_quantize, \ has_zstandard, has_bzip2, has_blosc, has_set_alignment, has_ncfilter = \ - check_api(inc_dirs,netcdf_lib_version) # for netcdf 4.4.x CDF5 format is always enabled. if netcdf_lib_version is not None and\ From baa3d3b7f7a93b91083ff61b25f08daeb9e85430 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 12:24:56 -0700 Subject: [PATCH 18/42] check mpi4py import --- .github/workflows/miniconda.yml | 1 + setup.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index f82deb57c..79c2ae2ff 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -72,6 +72,7 @@ jobs: export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config which nc-config nc-config --all + python -c "import mpi4py" pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda diff --git a/setup.py b/setup.py index 1fdd37f88..3de06408b 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,6 @@ def check_api(inc_dirs,netcdf_lib_version): has_parallel_support = bool(int(line.split()[2])) if line.startswith('#define NC_HAS_PARALLEL4'): has_parallel4_support = bool(int(line.split()[2])) - print('has_parallel4_support',has_parallel4_support) if line.startswith('#define NC_HAS_PNETCDF'): has_pnetcdf_support = bool(int(line.split()[2])) if line.startswith('#define NC_HAS_SZIP_WRITE'): @@ -572,8 +571,6 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): try: import mpi4py except ImportError: - if has_parallel4_support or has_pnetcdf_support: - f.write('no mpi4py found, disabling mpi parallel support\n') has_parallel4_support = False has_pnetcdf_support = False From db5c635289c8cc4f52373182c795652f9eb6ccf1 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 12:28:52 -0700 Subject: [PATCH 19/42] don't try to import mpi4py --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 3de06408b..434d27ca2 100644 --- a/setup.py +++ b/setup.py @@ -568,11 +568,11 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format = True # disable parallel support if mpi4py not available. - try: - import mpi4py - except ImportError: - has_parallel4_support = False - has_pnetcdf_support = False + #try: + # import mpi4py + #except ImportError: + # has_parallel4_support = False + # has_pnetcdf_support = False f = open(osp.join('include', 'constants.pyx'), 'w') if has_rename_grp: From 54fa91c711bb329c0b01fd48d902bb979c66e022 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 12:49:07 -0700 Subject: [PATCH 20/42] add --no-build-isolation to run-mpi test, since mpi4py not in pyproject.toml --- .github/workflows/miniconda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 79c2ae2ff..c1add3c33 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -73,7 +73,7 @@ jobs: which nc-config nc-config --all python -c "import mpi4py" - pip install -v -v -v -e . --no-deps --force-reinstall + pip install -v -v -v -e . --no-deps --no-build-isolation --force-reinstall - name: Debug conda shell: bash -l {0} From 1cbac5a9faafde739463342fe19b6063a0d778f7 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 12:55:09 -0700 Subject: [PATCH 21/42] update --- .github/workflows/miniconda.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index c1add3c33..95cab38b0 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -70,10 +70,8 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest mpi4py hdf5=*=mpi* libnetcdf=*=mpi* cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - which nc-config nc-config --all - python -c "import mpi4py" - pip install -v -v -v -e . --no-deps --no-build-isolation --force-reinstall + pip install -v . --no-deps --no-build-isolation --force-reinstall - name: Debug conda shell: bash -l {0} From d99ad14c2c7608586e4b3f7b103ed7d74c1dfc63 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:00:17 -0700 Subject: [PATCH 22/42] try moving pyproject.toml out of the way for mpi build --- .github/workflows/miniconda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 95cab38b0..d1202a58a 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,7 +71,8 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - pip install -v . --no-deps --no-build-isolation --force-reinstall + mv pyproject.toml pyproject.toml.save + pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From 02f3c1061f8dbbc43de65fcc26b26c195c798d15 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:03:09 -0700 Subject: [PATCH 23/42] update --- .github/workflows/miniconda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index d1202a58a..7c24e3710 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -72,7 +72,7 @@ jobs: export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all mv pyproject.toml pyproject.toml.save - pip install -v -v -v -e . --no-deps --force-reinstall + pip install -v -v -v --no-build-isolation -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From 3aa712b2a3b2ea8b82d03c743bc75d12c1dcf9f9 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:06:14 -0700 Subject: [PATCH 24/42] new file --- pyproject.toml.mpi | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml.mpi diff --git a/pyproject.toml.mpi b/pyproject.toml.mpi new file mode 100644 index 000000000..9d7e29b1b --- /dev/null +++ b/pyproject.toml.mpi @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=41.2", "cython>=0.19", "oldest-supported-numpy", "mpi4py"] +build-backend = "setuptools.build_meta" From d2a518b728b58411d8873e1305b2f6d5314f728f Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:06:57 -0700 Subject: [PATCH 25/42] try again with pyproject.toml that includes mpi4py --- .github/workflows/miniconda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 7c24e3710..445b0ed4a 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -72,7 +72,8 @@ jobs: export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all mv pyproject.toml pyproject.toml.save - pip install -v -v -v --no-build-isolation -e . --no-deps --force-reinstall + cp pyproject.toml.mpi pyproject.toml + pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From ba28488076ed732a794a7d3a8f2e04212dfe4660 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:10:31 -0700 Subject: [PATCH 26/42] revert previous change --- .github/workflows/miniconda.yml | 2 -- pyproject.toml.mpi | 3 --- 2 files changed, 5 deletions(-) delete mode 100644 pyproject.toml.mpi diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 445b0ed4a..d135934a0 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,8 +71,6 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - mv pyproject.toml pyproject.toml.save - cp pyproject.toml.mpi pyproject.toml pip install -v -v -v -e . --no-deps --force-reinstall - name: Debug conda diff --git a/pyproject.toml.mpi b/pyproject.toml.mpi deleted file mode 100644 index 9d7e29b1b..000000000 --- a/pyproject.toml.mpi +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools>=41.2", "cython>=0.19", "oldest-supported-numpy", "mpi4py"] -build-backend = "setuptools.build_meta" From f6373210bf6946a6dc640a69922f32901eecc4f3 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:11:50 -0700 Subject: [PATCH 27/42] update --- .github/workflows/miniconda.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index d135934a0..c5cb041eb 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,7 +71,9 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - pip install -v -v -v -e . --no-deps --force-reinstall + #pip install -v -v -v -e . --no-deps --force-reinstall + # don't use pip to install since mpi4py will not be found + python setup.py install - name: Debug conda shell: bash -l {0} From eacd6e6a644462a47bbe7dc86a40acd802fb4525 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:17:48 -0700 Subject: [PATCH 28/42] add mpi4py import --- setup.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 434d27ca2..25a11c265 100644 --- a/setup.py +++ b/setup.py @@ -568,11 +568,12 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format = True # disable parallel support if mpi4py not available. - #try: - # import mpi4py - #except ImportError: - # has_parallel4_support = False - # has_pnetcdf_support = False + try: + import mpi4py + except ImportError: + f.write('disabling mpi parallel support because mpi4py not found\n') + has_parallel4_support = False + has_pnetcdf_support = False f = open(osp.join('include', 'constants.pyx'), 'w') if has_rename_grp: @@ -684,6 +685,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): f.close() if has_parallel4_support or has_pnetcdf_support: + import mpi4py inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) From 171bf165dffdcc0e83fabeb8f3d40756b371d646 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:23:29 -0700 Subject: [PATCH 29/42] try --no-build-isolation again --- .github/workflows/miniconda.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index c5cb041eb..2040c4ce4 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,9 +71,10 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - #pip install -v -v -v -e . --no-deps --force-reinstall + python -c "import mpi4py, print('mpi4py version ',mpi4py.__version__)" + pip install -v -v -v --no-build-isolation -e . --no-deps --force-reinstall # don't use pip to install since mpi4py will not be found - python setup.py install + #python setup.py install - name: Debug conda shell: bash -l {0} From e64e36a7ebfd98e2f17fa4e2d80d0779f0aafe89 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:31:14 -0700 Subject: [PATCH 30/42] update --- .github/workflows/miniconda.yml | 3 --- setup.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 2040c4ce4..27ad6aa7c 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,10 +71,7 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - python -c "import mpi4py, print('mpi4py version ',mpi4py.__version__)" pip install -v -v -v --no-build-isolation -e . --no-deps --force-reinstall - # don't use pip to install since mpi4py will not be found - #python setup.py install - name: Debug conda shell: bash -l {0} diff --git a/setup.py b/setup.py index 25a11c265..ef6ee8f19 100644 --- a/setup.py +++ b/setup.py @@ -570,7 +570,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): # disable parallel support if mpi4py not available. try: import mpi4py - except ImportError: + except: f.write('disabling mpi parallel support because mpi4py not found\n') has_parallel4_support = False has_pnetcdf_support = False From 7e72caab33e6c280b9be774c92025f98d95148e6 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:33:40 -0700 Subject: [PATCH 31/42] update --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ef6ee8f19..b1013f04c 100644 --- a/setup.py +++ b/setup.py @@ -568,12 +568,12 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format = True # disable parallel support if mpi4py not available. - try: - import mpi4py - except: - f.write('disabling mpi parallel support because mpi4py not found\n') - has_parallel4_support = False - has_pnetcdf_support = False + #try: + # import mpi4py + #except: + # f.write('disabling mpi parallel support because mpi4py not found\n') + # has_parallel4_support = False + # has_pnetcdf_support = False f = open(osp.join('include', 'constants.pyx'), 'w') if has_rename_grp: From 4235343a502fe2eab27ac2394089bb184829caff Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 13:53:10 -0700 Subject: [PATCH 32/42] add --no-build-isolation to Tarball --- .github/workflows/build_latest.yml | 2 +- .github/workflows/build_old.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 10e7afd6a..e70426c2e 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -100,5 +100,5 @@ jobs: python setup.py --version check-manifest --version check-manifest --verbose - pip wheel . -w dist --no-deps + pip wheel . -w dist --no-build-isolation --no-deps twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index c4fe0ad29..d28d62eff 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -100,5 +100,5 @@ jobs: python setup.py --version check-manifest --version check-manifest --verbose - pip wheel . -w dist --no-deps + pip wheel . -w dist --no-build-isolation --no-deps twine check dist/* From 8160a9a17cfbe10f19930615b75582c1c4d16170 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 16:01:11 -0700 Subject: [PATCH 33/42] catch mpi4py ImportError --- setup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index b1013f04c..0ef10e2b5 100644 --- a/setup.py +++ b/setup.py @@ -568,12 +568,14 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format = True # disable parallel support if mpi4py not available. - #try: - # import mpi4py - #except: - # f.write('disabling mpi parallel support because mpi4py not found\n') - # has_parallel4_support = False - # has_pnetcdf_support = False + try: + import mpi4py + has_mpi4py = True + except ImportError: + f.write('disabling mpi parallel support because mpi4py not found\n') + has_parallel4_support = False + has_pnetcdf_support = False + has_mpi4py = False f = open(osp.join('include', 'constants.pyx'), 'w') if has_rename_grp: @@ -684,8 +686,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): f.close() - if has_parallel4_support or has_pnetcdf_support: - import mpi4py + if has_mpi4py and (has_parallel4_support or has_pnetcdf_support): inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) From 13f4a5bbad9eb8fd9874319fba2525e2c53ecdc4 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 16:02:02 -0700 Subject: [PATCH 34/42] update --- .github/workflows/build_latest.yml | 2 +- .github/workflows/build_old.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index e70426c2e..10e7afd6a 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -100,5 +100,5 @@ jobs: python setup.py --version check-manifest --version check-manifest --verbose - pip wheel . -w dist --no-build-isolation --no-deps + pip wheel . -w dist --no-deps twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index d28d62eff..c4fe0ad29 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -100,5 +100,5 @@ jobs: python setup.py --version check-manifest --version check-manifest --verbose - pip wheel . -w dist --no-build-isolation --no-deps + pip wheel . -w dist --no-deps twine check dist/* From 34cc9b964564ff10d8b2c6a895a50817c0080b9f Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 16:17:06 -0700 Subject: [PATCH 35/42] disable tarball step --- .github/workflows/build_latest.yml | 16 ++++++++-------- .github/workflows/build_old.yml | 16 ++++++++-------- setup.py | 16 +++++++--------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 10e7afd6a..3e9e0e448 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -94,11 +94,11 @@ jobs: echo "pnetcdf mpi test passed!" fi - - name: Tarball - run: | - export PATH=${NETCDF_DIR}/bin:${PATH} - python setup.py --version - check-manifest --version - check-manifest --verbose - pip wheel . -w dist --no-deps - twine check dist/* +# - name: Tarball +# run: | +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version +# check-manifest --version +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index c4fe0ad29..3bc79d718 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -94,11 +94,11 @@ jobs: echo "pnetcdf mpi test passed!" fi - - name: Tarball - run: | - export PATH=${NETCDF_DIR}/bin:${PATH} - python setup.py --version - check-manifest --version - check-manifest --verbose - pip wheel . -w dist --no-deps - twine check dist/* +# - name: Tarball +# run: | +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version +# check-manifest --version +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* diff --git a/setup.py b/setup.py index 0ef10e2b5..7355a617d 100644 --- a/setup.py +++ b/setup.py @@ -568,14 +568,12 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): has_cdf5_format = True # disable parallel support if mpi4py not available. - try: - import mpi4py - has_mpi4py = True - except ImportError: - f.write('disabling mpi parallel support because mpi4py not found\n') - has_parallel4_support = False - has_pnetcdf_support = False - has_mpi4py = False + #try: + # import mpi4py + #except ImportError: + # f.write('disabling mpi parallel support because mpi4py not found\n') + # has_parallel4_support = False + # has_pnetcdf_support = False f = open(osp.join('include', 'constants.pyx'), 'w') if has_rename_grp: @@ -686,7 +684,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): f.close() - if has_mpi4py and (has_parallel4_support or has_pnetcdf_support): + if has_parallel4_support or has_pnetcdf_support: inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) From d4001e53d7c9aa8d1e06a7c61c1718b002d11bc9 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 16:40:36 -0700 Subject: [PATCH 36/42] update --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7355a617d..7a88e23c9 100644 --- a/setup.py +++ b/setup.py @@ -685,6 +685,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): f.close() if has_parallel4_support or has_pnetcdf_support: + import mpi4py inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) From 82f4acbb90cd856cbd8b8a8ef10a026107887945 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 17:57:49 -0700 Subject: [PATCH 37/42] test setting MPI4PY_INCLUDE in environment --- .github/workflows/miniconda.yml | 3 ++- setup.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 27ad6aa7c..b84c8ab75 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,7 +71,8 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - pip install -v -v -v --no-build-isolation -e . --no-deps --force-reinstall + export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` + pip install -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} diff --git a/setup.py b/setup.py index 7a88e23c9..f8c37eae1 100644 --- a/setup.py +++ b/setup.py @@ -685,8 +685,11 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): f.close() if has_parallel4_support or has_pnetcdf_support: - import mpi4py - inc_dirs.append(mpi4py.get_include()) + if os.environ.get('MPI4PY_INCLUDE'): + inc_dirs.append(os.environ.get('MPI4PY_INCLUDE')) + else: + import mpi4py + inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) if mpi_incdir is not None: inc_dirs.append(mpi_incdir) From 854031000bc245966f3afb37b652546d680d0c67 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 18:19:51 -0700 Subject: [PATCH 38/42] add debug prints --- .github/workflows/miniconda.yml | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index b84c8ab75..dab0768aa 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -72,6 +72,7 @@ jobs: export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` + echo "MPI4PY_INCLUDE=${MPI4PY_INCLUDE}" pip install -v -e . --no-deps --force-reinstall - name: Debug conda diff --git a/setup.py b/setup.py index f8c37eae1..185c347d0 100644 --- a/setup.py +++ b/setup.py @@ -693,6 +693,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) if mpi_incdir is not None: inc_dirs.append(mpi_incdir) + print('inc_dirs=',inc_dirs) ext_modules = [Extension("netCDF4._netCDF4", [netcdf4_src_pyx], From fb9cdc7800df0ee5de5934f0b1b2b0a315c78d44 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 18:25:25 -0700 Subject: [PATCH 39/42] try again --- .github/workflows/miniconda.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index dab0768aa..ad0fb42df 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,9 +71,8 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` - echo "MPI4PY_INCLUDE=${MPI4PY_INCLUDE}" - pip install -v -e . --no-deps --force-reinstall + #export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` + pip install -v -e . --no-build-isolation --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From 6cba9d216075d20d922dc38e300f68d2ee95cd53 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 18:30:16 -0700 Subject: [PATCH 40/42] update --- .github/workflows/miniconda.yml | 3 ++- setup.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index ad0fb42df..61a5ad402 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,7 +71,8 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - #export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` + #pip install -v -e . --no-deps --force-reinstall + export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` pip install -v -e . --no-build-isolation --no-deps --force-reinstall - name: Debug conda diff --git a/setup.py b/setup.py index 185c347d0..f8c37eae1 100644 --- a/setup.py +++ b/setup.py @@ -693,7 +693,6 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) if mpi_incdir is not None: inc_dirs.append(mpi_incdir) - print('inc_dirs=',inc_dirs) ext_modules = [Extension("netCDF4._netCDF4", [netcdf4_src_pyx], From e0fed04f8bc8f3161607a9e0f6cfdc9513bc8fc8 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 18:37:25 -0700 Subject: [PATCH 41/42] try again --- .github/workflows/build_latest.yml | 17 +++++++++-------- .github/workflows/build_old.yml | 17 +++++++++-------- .github/workflows/miniconda.yml | 6 +++--- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index 3e9e0e448..e2e473cb4 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -94,11 +94,12 @@ jobs: echo "pnetcdf mpi test passed!" fi -# - name: Tarball -# run: | -# export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version -# check-manifest --version -# check-manifest --verbose -# pip wheel . -w dist --no-deps -# twine check dist/* + - name: Tarball + run: | + export PATH=${NETCDF_DIR}/bin:${PATH} + python setup.py --version + check-manifest --version + check-manifest --verbose + export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` + pip wheel . -w dist --no-deps + twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index 3bc79d718..cff34a3bb 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -94,11 +94,12 @@ jobs: echo "pnetcdf mpi test passed!" fi -# - name: Tarball -# run: | -# export PATH=${NETCDF_DIR}/bin:${PATH} -# python setup.py --version -# check-manifest --version -# check-manifest --verbose -# pip wheel . -w dist --no-deps -# twine check dist/* + - name: Tarball + run: | + export PATH=${NETCDF_DIR}/bin:${PATH} + python setup.py --version + check-manifest --version + check-manifest --verbose + export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` + pip wheel . -w dist --no-deps + twine check dist/* diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index 61a5ad402..f8f0a708a 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -34,7 +34,7 @@ jobs: micromamba create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime zlib --channel conda-forge micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config - pip install -v -v -v -e . --no-deps --force-reinstall + pip install -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} @@ -71,9 +71,9 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - #pip install -v -e . --no-deps --force-reinstall + #pip install -v -e . --no-build-isolation --no-deps --force-reinstall export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` - pip install -v -e . --no-build-isolation --no-deps --force-reinstall + pip install -v -e . --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} From c5b630ae1d69ada15f2d17753f30c3559dab1a53 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 12 Nov 2022 18:46:03 -0700 Subject: [PATCH 42/42] revert --- .github/workflows/build_latest.yml | 17 ++++++++--------- .github/workflows/build_old.yml | 17 ++++++++--------- .github/workflows/miniconda.yml | 4 +--- setup.py | 7 ++----- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build_latest.yml b/.github/workflows/build_latest.yml index e2e473cb4..3e9e0e448 100644 --- a/.github/workflows/build_latest.yml +++ b/.github/workflows/build_latest.yml @@ -94,12 +94,11 @@ jobs: echo "pnetcdf mpi test passed!" fi - - name: Tarball - run: | - export PATH=${NETCDF_DIR}/bin:${PATH} - python setup.py --version - check-manifest --version - check-manifest --verbose - export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` - pip wheel . -w dist --no-deps - twine check dist/* +# - name: Tarball +# run: | +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version +# check-manifest --version +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* diff --git a/.github/workflows/build_old.yml b/.github/workflows/build_old.yml index cff34a3bb..3bc79d718 100644 --- a/.github/workflows/build_old.yml +++ b/.github/workflows/build_old.yml @@ -94,12 +94,11 @@ jobs: echo "pnetcdf mpi test passed!" fi - - name: Tarball - run: | - export PATH=${NETCDF_DIR}/bin:${PATH} - python setup.py --version - check-manifest --version - check-manifest --verbose - export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` - pip wheel . -w dist --no-deps - twine check dist/* +# - name: Tarball +# run: | +# export PATH=${NETCDF_DIR}/bin:${PATH} +# python setup.py --version +# check-manifest --version +# check-manifest --verbose +# pip wheel . -w dist --no-deps +# twine check dist/* diff --git a/.github/workflows/miniconda.yml b/.github/workflows/miniconda.yml index f8f0a708a..042bb1625 100644 --- a/.github/workflows/miniconda.yml +++ b/.github/workflows/miniconda.yml @@ -71,9 +71,7 @@ jobs: micromamba activate TEST export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config nc-config --all - #pip install -v -e . --no-build-isolation --no-deps --force-reinstall - export MPI4PY_INCLUDE=`python -c 'import mpi4py; print(mpi4py.get_include())'` - pip install -v -e . --no-deps --force-reinstall + pip install -v -e . --no-build-isolation --no-deps --force-reinstall - name: Debug conda shell: bash -l {0} diff --git a/setup.py b/setup.py index f8c37eae1..7a88e23c9 100644 --- a/setup.py +++ b/setup.py @@ -685,11 +685,8 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs): f.close() if has_parallel4_support or has_pnetcdf_support: - if os.environ.get('MPI4PY_INCLUDE'): - inc_dirs.append(os.environ.get('MPI4PY_INCLUDE')) - else: - import mpi4py - inc_dirs.append(mpi4py.get_include()) + import mpi4py + inc_dirs.append(mpi4py.get_include()) # mpi_incdir should not be needed if using nc-config # (should be included in nc-config --cflags) if mpi_incdir is not None: inc_dirs.append(mpi_incdir)