/
create_conda_software_bundle.sh
executable file
·151 lines (130 loc) · 6.05 KB
/
create_conda_software_bundle.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#!/usr/bin/env bash
# Create a software bundle from a conda environment
# Usage: ./create_conda_software_bundle.sh <bundle directory>
# Creates a software bundle directory and a tarball of that directory
#SHELLB3_DEFAULT="ftp://ftp.ssec.wisc.edu/pub/shellb3/ShellB3-Linux-x86_64-20140212-r840-core-cspp.tar.gz"
BASE_P2G_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PY_DIR="$BASE_P2G_DIR"
BUNDLE_SCRIPTS_DIR="$BASE_P2G_DIR"/swbundle
CACHE_DIR="/data/tmp"
oops() {
echo "OOPS: $*"
echo "FAILURE"
exit 1
}
cached_download() {
fn=`basename $1`
cache_path="${CACHE_DIR}/$fn"
pushd $CACHE_DIR
if [ ! -f $cache_path ]; then
wget $1 || oops "Could not download $1"
fi
popd
cp $cache_path . || oops "Could not copy cached download ${cache_path}"
}
# This assumes that the current conda environment is already active
which conda || oops "Conda environment must be available"
which conda-pack || oops "'conda-pack' must be installed in conda environment"
if [ $# -eq 1 ]; then
SB_NAME=$1
else
echo "ERROR: Invalid Arguments"
echo "Usage: ./create_conda_software_bundle.sh <bundle name>"
exit 1
fi
sb_base_name=$(basename $SB_NAME)
if [[ $sb_base_name == *"polar"* ]]; then
PROJECT="P2G"
else
PROJECT="G2G"
fi
# If SB_NAME is a relative path, make it an absolute path.
if [[ "$SB_NAME" != /* ]]; then
SB_NAME="$(pwd)/${SB_NAME}"
fi
pip install -U --no-deps "$BASE_P2G_DIR" || oops "Couldn't install current polar2grid package"
SB_TARBALL="${SB_NAME}.tar.gz"
conda clean -afy
conda-pack --n-threads -1 --compress-level 0 -o $SB_TARBALL || oops "Couldn't create conda-packed tarball"
PYTHON_RUNTIME_BASE="${SB_NAME}/libexec/python_runtime"
mkdir -p ${PYTHON_RUNTIME_BASE} || oops "Couldn't make output directory"
tar -xzf ${SB_TARBALL} -C ${PYTHON_RUNTIME_BASE} || oops "Couldn't untar conda-packed tarball"
cd ${SB_NAME} || oops "Couldn't change to software bundle directory"
echo "Copying user grid directory to software bundle"
cp -r ${BUNDLE_SCRIPTS_DIR}/grid_configs .
cp -r ${BUNDLE_SCRIPTS_DIR}/example_enhancements .
mkdir -p gshhg_data || oops "Could not make GSHHG data directory"
pushd gshhg_data
echo "Downloading GSHHG shapefiles"
cached_download https://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.6.zip
unzip gshhg-shp-2.3.6.zip || oops "Could not unpack GSHHG shapefiles"
rm gshhg-shp-2.3.6.zip || oops "Could not delete the GSHHG zip file"
chmod 444 `find . -type f` || oops "Could not make GSHHG shapefiles readable by everyone"
popd
echo "Copying bash scripts to software bundle bin"
cd "${SB_NAME}"
mkdir -p bin || oops "Couldn't make 'bin' directory"
mkdir -p etc || oops "Couldn't make 'etc' directory"
# expand glob pattern
P2G_ETC_DIR=$(echo libexec/python_runtime/lib/python?.??/site-packages/polar2grid/etc)
if [ ! -d "${P2G_ETC_DIR}" ]; then
oops "Couldn't find polar2grid etc directory"
fi
echo "Package etc directory found at: ${P2G_ETC_DIR}"
ln -s ../${P2G_ETC_DIR} etc/polar2grid || oops "Couldn't link to package etc directory"
ln -s etc/polar2grid/colormaps . || oops "Couldn't create softlink for colormaps directory"
cp -P ${BUNDLE_SCRIPTS_DIR}/*.sh ${BUNDLE_SCRIPTS_DIR}/*.txt bin/ || echo "Couldn't copy scripts to bin/ directory"
# clean up readmes and add release notes
if [[ $PROJECT == "P2G" ]]; then
rm -f bin/GEO2GRID_README.txt
mv bin/POLAR2GRID_README.txt README.txt
cp $BASE_P2G_DIR/NEWS.rst $SB_NAME/RELEASE_NOTES.txt || oops "Couldn't copy release notes to destination directory"
else
rm -f bin/POLAR2GRID_README.txt
mv bin/GEO2GRID_README.txt README.txt
cp $BASE_P2G_DIR/NEWS_GEO2GRID.rst $SB_NAME/RELEASE_NOTES.txt || oops "Couldn't copy release notes to destination directory"
fi
# Download pyspectral data
echo "Downloading pyspectral data..."
PSP_DATA_CACHE_ROOT="${CACHE_DIR}/p2g_pyspectral_cache"
mkdir -p "${PSP_DATA_CACHE_ROOT}" || oops "Couldn't create pyspectral data cache"
PSP_CONFIG_FILE=etc/polar2grid/pyspectral.yaml PSP_DATA_ROOT=pyspectral_data PSP_DATA_CACHE_ROOT=${PSP_DATA_CACHE_ROOT} $SB_NAME/bin/download_pyspectral_data.sh || oops "Couldn't download pyspectral data"
# Download Satpy auxiliary data
echo "Downloading Satpy auxiliary data..."
AUX_CACHE_DIR="${CACHE_DIR}/satpy_aux_data_${USER}"
SATPY_DATA_DIR="${SB_NAME}/share/polar2grid/data"
SATPY_CONFIG_PATH="${P2G_ETC_DIR}" \
${PYTHON_RUNTIME_BASE}/bin/satpy_retrieve_all_aux_data \
--data-dir ${AUX_CACHE_DIR} || oops "Could not download Satpy auxiliary data"
# Add the download_from_internet: False to the config
echo "download_from_internet: False" >> ${SB_NAME}/etc/polar2grid/pyspectral.yaml
# Inject environment code into swbundle only.
# Do this *after* the above scripts are called to avoid running the one-time setup of the swbundle
cd $SB_NAME/bin
for bash_file in *.sh; do
sed -i "s/# __SWBUNDLE_ENVIRONMENT_INJECTION__/source \$POLAR2GRID_HOME\/bin\/env.sh/g" "$bash_file"
done
# Softlink bin/ scripts in python runtime so env.sh adds them to PATH
cd ${PYTHON_RUNTIME_BASE}/bin
find ../../../bin/ -name "*.sh" ! -name "*env*" -exec ln -s {} . \;
echo "Copying Satpy auxiliary data to software bundle..."
mkdir -p ${SATPY_DATA_DIR} || oops "Could not create polar2grid auxiliary data directory"
# don't include large geotiff files that we don't use in P2G/G2G
rsync -auv --exclude "*.tif" ${AUX_CACHE_DIR}/* "${SATPY_DATA_DIR}/" || oops "Could not copy Satpy auxiliary data to software bundle"
# Perform extra "risky" operations to make the tarball as small as possible
# Taken from https://jcrist.github.io/conda-docker-tips.html
MINIFY_TARBALL=${MINIFY_TARBALL:-1}
if [ $MINIFY_TARBALL -ne 0 ]; then
cd $SB_NAME
find . -follow -type f -name '*.a' -delete
find . -follow -type f -name '*.pyc' -delete
find . -follow -type f -name '*.js.map' -delete
# Disable this as conda-unpack fails
#find ${PYTHON_RUNTIME_BASE}/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
fi
# Tar up the software bundle
echo "Creating software bundle tarball..."
cd "$SB_NAME"/..
rm -f ${SB_TARBALL}
tar -czf "$SB_TARBALL" "$(basename "$SB_NAME")"
echo "SUCCESS"