Merge pull request #5 from Noah-Kennedy/dev

Dev
Noah-Kennedy · Nov 2, 2020 · ed23596 · ed23596
2 parents fbda0fc + f4b13c1
commit ed23596
Show file tree

Hide file tree

Showing 6 changed files with 133 additions and 69 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,4 +10,5 @@
 *.bip
 *.bil
 
-/.cargo
+/.cargo
+/bench-data
diff --git a/Cargo.toml b/Cargo.toml
@@ -12,17 +12,15 @@ structopt = "0.3.18"
 # Regex library
 regex = "1.4"
 
-# Fork of memmap2 library. I added support for MMAP_POPULATE, to enable page cache preloading.
-[dependencies.memmap2]
-git = "https://github.com/Noah-Kennedy/memmap2-rs"
+# Provides numeric traits and generic numeric functions
+num = "0.3.0"
 
 # Provides status bar
-[dependencies.indicatif]
-version = "0.15.0"
+indicatif = "0.15.0"
 
-# Provides numeric traits and generic numeric functions
-[dependencies.num]
-version = "0.3.0"
+# Fork of memmap2 library. I added support for MMAP_POPULATE, to enable page cache preloading.
+[dependencies.memmap2]
+git = "https://github.com/Noah-Kennedy/memmap2-rs"
 
 # Image processing library, used to generate PNG images.
 [dependencies.image]

diff --git a/README.md b/README.md
@@ -1,5 +1,7 @@
-# Hyperspectral
-TODO overview
+# Hyperspectra
+
+`hyperspectra` is a cli tool for manipulating ENVI BIP, BIL, and BSQ files for processing
+hyperspectral image data.
 
 ## Installation
 
@@ -53,6 +55,9 @@ TODO
 ## Benchmarks
 Benchmarks were performed on a machine running Arch Linux with a Ryzen 3900X, an Nvidia RTX 2080 TI,
 64 GiB of Ram, and SSD storage.
+The machine was running a Linux 5.4 LTS kernel release.
+Some testing (not formally benchmarked yet) shows that this <b>might</b> run faster on a 5.8+ kernel
+ir with `mitigations=off` in `grub.cfg`.
 
 Input files had 5 bands and were 28740 x 21954.
 
@@ -63,51 +68,61 @@ in the cache when the run began.
 
 ### Conversion
 #### Warm Cache
-|Input|Output|Time (mean ± σ)|
-|-----|------|----|
-|bip|bil|24.469 s ± 0.815 s|
-|bip|bsq|9.458 s ± 0.429 s|
-|bil|bip|24.576 s ± 1.149 s|
-|bil|bsq|11.683 s ± 0.246 s|
-|bsq|bip|24.582 s ± 0.431 s|
-|bsq|bil|24.679 s ± 0.811 s|
+| Input| Output | Mean [s] | Min [s] | Max [s] |
+|:---|:---|---:|---:|---:|
+| bip | bip | 16.744 ± 0.407 | 16.070 | 17.234 |
+| bip | bil | 14.597 ± 0.429 | 13.714 | 15.171 |
+| bip | bsq | 14.504 ± 0.396 | 13.639 | 14.944 |
+| bil | bip | 12.945 ± 0.859 | 11.918 | 14.283 |
+| bil | bil | 14.146 ± 0.399 | 13.631 | 14.730 |
+| bil | bsq | 13.746 ± 0.874 | 12.111 | 14.829 |
+| bsq | bip | 10.918 ± 0.458 | 10.395 | 11.771 |
+| bsq | bil | 13.978 ± 0.554 | 12.851 | 14.742 |
+| bsq | bsq | 13.941 ± 0.747 | 12.380 | 15.100 |
 
 #### Cold Cache
-|Input|Output|Time (mean ± σ)|
-|-----|------|----|
-|bip|bil||
-|bip|bsq||
-|bil|bip||
-|bil|bsq|58.449 s ±  0.254 s|
-|bsq|bip|56.836 s ± 0.577 s|
-|bsq|bil|58.137 s ± 0.660 s|
+| Input| Output | Mean [s] | Min [s] | Max [s] |
+|:---|:---|---:|---:|---:|
+| bip | bip | 39.153 ± 1.239 | 37.339 | 42.041 |
+| bip | bil | 37.684 ± 0.648 | 36.101 | 38.228 |
+| bip | bsq | 37.709 ± 0.210 | 37.399 | 38.023 |
+| bil | bip | 36.161 ± 0.980 | 35.244 | 38.292 |
+| bil | bil | 37.131 ± 0.426 | 36.210 | 37.637 |
+| bil | bsq | 37.029 ± 0.266 | 36.659 | 37.382 |
+| bsq | bip | 34.321 ± 0.396 | 33.436 | 34.706 |
+| bsq | bil | 36.851 ± 0.395 | 36.236 | 37.493 |
+| bsq | bsq | 36.993 ± 0.244 | 36.629 | 37.259 |
 
 ### Colorization
 #### Warm Cache
-|Input|Color Map|Time (mean ± σ)   |
-|-----|---------|------------------|
-|bip  |RGB      |28.259 s ± 0.691 s|
-|bil  |RGB      |27.717 s ± 0.669 s|
-|bsq  |RGB      |27.594 s ± 0.154 s|
-|bip  |Coolwarm |20.462 s ± 1.040 s|
-|bil  |Coolwarm |20.451 s ± 1.089 s|
-|bsq  |Coolwarm |20.101 s ± 0.410 s|
-|bip  |Gray     |11.781 s ± 0.137 s|
-|bil  |Gray     |11.877 s ± 0.028 s|
-|bsq  |Gray     |11.937 s ± 0.170 s|
+| Input| Output | Mean [s] | Min [s] | Max [s] |
+|:---|:---|---:|---:|---:|
+| bip | rgb         | 23.540 ± 0.052 | 23.478 | 23.642 |
+| bip | coolwarm    | 17.000 ± 1.101 | 16.488 | 20.015 |
+| bip | grey        | 8.746 ± 0.196  |  8.647 |  9.292 |
+| bil | rgb         | 23.251 ± 0.040 | 23.197 | 23.329 |
+| bil | coolwarm    | 16.711 ± 1.206 | 16.188 | 20.060 |
+| bil | grey        | 8.878 ± 0.079  |  8.832 |  9.094 |
+| bsq | rgb         | 23.405 ± 0.530 | 23.187 | 24.911 |
+| bsq | coolwarm    | 16.758 ± 1.245 | 16.142 | 20.089 |
+| bsq | grey        | 8.995 ± 0.308  |  8.823 |  9.625 |
 
 #### Cold Cache
-|Input|Color Map|Time (mean ± σ)   |
-|-----|---------|------------------|
-|bip  |RGB      |56.159 s ± 1.152 s|
-|bil  |RGB      |55.467 s ± 0.470 s|
-|bsq  |RGB      |54.522 s ± 0.392 s|
-|bip  |Coolwarm |43.794 s ± 0.388 s|
-|bil  |Coolwarm |43.282 s ± 0.440 s|
-|bsq  |Coolwarm |43.353 s ± 0.341 s|
-|bip  |Gray     |35.436 s ± 0.274 s|
-|bil  |Gray     |34.771 s ± 0.280 s|
-|bsq  |Gray     |34.259 s ± 0.161 s|
+| Input| Output | Mean [s] | Min [s] | Max [s] |
+|:---|:---|---:|---:|---:|
+| bip | rgb      | 46.864 ± 0.686 | 46.472 | 48.173 |
+| bip | coolwarm | 40.113 ± 1.171 | 39.440 | 43.336 |
+| bip | grey     | 31.969 ± 0.511 | 31.626 | 33.259 |
+| bil | rgb      | 46.203 ± 0.051 | 46.131 | 46.262 |
+| bil | coolwarm | 39.165 ± 0.050 | 39.087 | 39.248 |
+| bil | grey     | 31.880 ± 0.380 | 31.694 | 32.953 |
+| bsq | rgb      | 46.297 ± 0.243 | 46.110 | 46.926 |
+| bsq | coolwarm | 39.181 ± 0.080 | 39.053 | 39.322 |
+| bsq | grey     | 32.010 ± 0.323 | 31.743 | 32.497 |
 
 ## Testing
-TODO
+Tests can be run by invoking `cargo test` on the command line.
+
+Benchmarks currently look for files in `./bench-data/` named `small.bil`, `small.bip`, `small.bsq`,
+`small.bil.hdr`, `small.bip.hdr`, `small.bsq.hdr` and will delete any png, bip, bil, or bsq files
+in the directory in which the benchmarks are run. Benchmarks use the `hyperfine` tool.
diff --git a/benchmark.sh b/benchmark.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+COLOR='hyperspectra color -i data/raw/unnormalized/unnorm.{type} -n data/raw/unnormalized/unnorm.{type}.hdr'
+
+RGB='-o rgb.png -m 0 0 0 -x 0.5 0.5 1 -b 1 3 4 -c rgb'
+COOL='-o cool.png -m 0 -x 1 -b 3 -c coolwarm'
+GREY='-o grey.png -m 0 -x 1 -b 3 -c grey'
+
+CONVERT_IN='hyperspectra convert -i data/raw/unnormalized/unnorm.{type} -n data/raw/unnormalized/unnorm.{type}.hdr'
+
+CONVERT_OUT_BIP='-o out.bip -t bip'
+CONVERT_OUT_BIL='-o out.bil -t bil'
+CONVERT_OUT_BSQ='-o out.bsq -t bsq'
+
+time hyperfine --warmup=2 \
+  --prepare 'rm *.png *.bil *.bip *.bsq || true' \
+  --export-markdown BENCHMARKS_CONVERT_WARM.md \
+  -L type bip,bil,bsq \
+  "$CONVERT_IN $CONVERT_OUT_BIP" \
+  "$CONVERT_IN $CONVERT_OUT_BIL" \
+  "$CONVERT_IN $CONVERT_OUT_BSQ"
+
+time hyperfine --warmup=2 \
+  --prepare 'rm *.png *.bil *.bip *.bsq || true' \
+  --export-markdown BENCHMARKS_COLOR_WARM.md \
+  -L type bip,bil,bsq \
+  "$COLOR $RGB" \
+  "$COLOR $COOL" \
+  "$COLOR $GREY"
+
+time hyperfine --warmup=2 \
+  --prepare 'rm *.png *.bil *.bip *.bsq || true; sync; echo 3 | tee /proc/sys/vm/drop_caches' \
+  --export-markdown BENCHMARKS_CONVERT_COLD.md \
+  -L type bip,bil,bsq \
+  "$CONVERT_IN $CONVERT_OUT_BIP" \
+  "$CONVERT_IN $CONVERT_OUT_BIL" \
+  "$CONVERT_IN $CONVERT_OUT_BSQ"
+
+time hyperfine --warmup=2 \
+  --prepare 'rm *.png *.bil *.bip *.bsq || true; sync; echo 3 | tee /proc/sys/vm/drop_caches' \
+  --export-markdown BENCHMARKS_COLOR_COLD.md \
+  -L type bip,bil,bsq \
+  "$COLOR $RGB" \
+  "$COLOR $COOL" \
+  "$COLOR $GREY"
+
+rm *.png *.bil *.bip *.bsq || true
+
+chown noah *.md
+chmod 755 *.md
diff --git a/src/bin_formats/file_buffer.rs b/src/bin_formats/file_buffer.rs
@@ -41,7 +41,7 @@ impl<T> FileInner<Mmap, T> {
 
         let raw = MmapOptions::new()
             .offset(headers.header_offset as u64)
-            .populate(true)
+            // .populate(true)
             .len(headers.bands * headers.samples * headers.lines * mem::size_of::<T>())
             .map(&file)?;
 
@@ -55,7 +55,7 @@ impl<T> FileInner<Mmap, T> {
     pub unsafe fn _from_dims(dims: &FileDims, file: &File) -> Result<Self, Box<dyn Error>> {
         let raw = MmapOptions::new()
             .offset(0)
-            .populate(true)
+            // .populate(true)
             .len(dims.bands.len() * dims.samples * dims.lines * mem::size_of::<T>())
             .map(&file)?;
 
@@ -84,12 +84,12 @@ impl<T> FileInner<MmapMut, T> {
         })
     }
 
-    pub unsafe fn headers_copy(headers: &Headers, file: &File) -> Result<Self, Box<dyn Error>> {
+    pub unsafe fn _headers_copy(headers: &Headers, file: &File) -> Result<Self, Box<dyn Error>> {
         assert_eq!(FileByteOrder::Intel, headers.byte_order);
 
         let raw = MmapOptions::new()
             .offset(headers.header_offset as u64)
-            .populate(true)
+            // .populate(true)
             .len(headers.bands * headers.samples * headers.lines * mem::size_of::<T>())
             .map_copy(&file)?;
 
@@ -105,7 +105,7 @@ impl<T> FileInner<MmapMut, T> {
 
         let raw = MmapOptions::new()
             .offset(headers.header_offset as u64)
-            .populate(true)
+            // .populate(true)
             .len(headers.bands * headers.samples * headers.lines * mem::size_of::<T>())
             .map_anon()?;
 
@@ -119,7 +119,7 @@ impl<T> FileInner<MmapMut, T> {
     pub unsafe fn _from_dims_mut(dims: &FileDims, file: &File) -> Result<Self, Box<dyn Error>> {
         let raw = MmapOptions::new()
             .offset(0)
-            .populate(true)
+            // .populate(true)
             .len(dims.bands.len() * dims.samples * dims.lines * mem::size_of::<T>())
             .map_mut(&file)?;
 
@@ -133,7 +133,7 @@ impl<T> FileInner<MmapMut, T> {
     pub unsafe fn _from_dims_copy(dims: &FileDims, file: &File) -> Result<Self, Box<dyn Error>> {
         let raw = MmapOptions::new()
             .offset(0)
-            .populate(true)
+            // .populate(true)
             .len(dims.bands.len() * dims.samples * dims.lines * mem::size_of::<T>())
             .map_copy(&file)?;
 
@@ -147,7 +147,7 @@ impl<T> FileInner<MmapMut, T> {
     pub unsafe fn _from_dims_anon(dims: &FileDims) -> Result<Self, Box<dyn Error>> {
         let raw = MmapOptions::new()
             .offset(0)
-            .populate(true)
+            // .populate(true)
             .len(dims.bands.len() * dims.samples * dims.lines * mem::size_of::<T>())
             .map_anon()?;
 

diff --git a/src/norm.rs b/src/norm.rs
@@ -5,7 +5,7 @@ use std::str::FromStr;
 
 use image::{GrayImage, RgbImage};
 
-use crate::bin_formats::{FileIndex, FileIndexMut, FileInner, Mat};
+use crate::bin_formats::{FileIndex, FileInner, Mat};
 use crate::bin_formats::bil::Bil;
 use crate::bin_formats::bip::Bip;
 use crate::bin_formats::bsq::Bsq;
@@ -21,27 +21,27 @@ pub fn normalize(opt: ColorOpt) -> Result<(), Box<dyn Error>> {
     let parsed_headers = Headers::from_str(&headers_str)?;
 
     println!("Mapping input file");
-    let inner: FileInner<_, f32> = unsafe { FileInner::headers_copy(&parsed_headers, &input_file)? };
+    let inner: FileInner<_, f32> = unsafe { FileInner::headers(&parsed_headers, &input_file)? };
 
     match parsed_headers.interleave {
         Interleave::Bip => {
-            let mut input = Mat::from(Bip::from(inner));
-            helper(&mut input, opt.output, &opt.color_map, &opt.min, &opt.max, &opt.bands)
+            let input = Mat::from(Bip::from(inner));
+            helper(&input, opt.output, &opt.color_map, &opt.min, &opt.max, &opt.bands)
         }
         Interleave::Bil => {
-            let mut input = Mat::from(Bil::from(inner));
-            helper(&mut input, opt.output, &opt.color_map, &opt.min, &opt.max, &opt.bands)
+            let input = Mat::from(Bil::from(inner));
+            helper(&input, opt.output, &opt.color_map, &opt.min, &opt.max, &opt.bands)
         }
         Interleave::Bsq => {
-            let mut input = Mat::from(Bsq::from(inner));
-            helper(&mut input, opt.output, &opt.color_map, &opt.min, &opt.max, &opt.bands)
+            let input = Mat::from(Bsq::from(inner));
+            helper(&input, opt.output, &opt.color_map, &opt.min, &opt.max, &opt.bands)
         }
     }
 }
 
-fn helper<F>(input: &mut Mat<f32, F>, path: PathBuf, f: &str, min: &[f32], max: &[f32], bands: &[usize])
+fn helper<F>(input: &Mat<f32, F>, path: PathBuf, f: &str, min: &[f32], max: &[f32], bands: &[usize])
              -> Result<(), Box<dyn Error>>
-    where F: 'static + FileIndex<f32> + FileIndexMut<f32> + Sync + Send
+    where F: 'static + FileIndex<f32> + Sync + Send
 {
     let (height, width, _) = input.inner.size();
 
@@ -80,7 +80,7 @@ fn helper<F>(input: &mut Mat<f32, F>, path: PathBuf, f: &str, min: &[f32], max:
         }
         "gray" | "grey" => {
             println!("Allocating output buffer");
-            let mut out = GrayImage::from_raw(
+            let mut out = GrayImage::from_raw(Moved benchmark files to new dir.
                 width as u32,
                 height as u32,
                 vec![0; height * width],