20120206-HPC3.tex

% \documentclass[handout]{beamer}
\documentclass{beamer}

\mode<presentation>
{
  \usetheme{default}
  \usefonttheme[onlymath]{serif}
  % \usetheme{Singapore}
  % \usetheme{Warsaw}
  % \usetheme{Malmoe}
  % \useinnertheme{circles}
  % \useoutertheme{infolines}
  % \useinnertheme{rounded}

  \setbeamercovered{transparent=5}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage{textpos,alltt,listings,multirow,ulem,siunitx}
\usepackage{pdfpages}
\newcommand\hmmax{0}
\newcommand\bmmax{0}
\usepackage{bm}

% font definitions, try \usepackage{ae} instead of the following
% three lines if you don't like this look
\usepackage{mathptmx}
\usepackage[scaled=.90]{helvet}
% \usepackage{courier}
\usepackage[T1]{fontenc}
\usepackage{tikz}
\usetikzlibrary[shapes,shapes.arrows,arrows,shapes.misc,fit,positioning]

% \usepackage{pgfpages}
% \pgfpagesuselayout{4 on 1}[a4paper,landscape,border shrink=5mm]

\usepackage{JedMacros}

\title{Toward less synchronous composable multilevel methods for implicit multiphysics simulation}
\author{Jed Brown\inst{1}, Mark Adams\inst{2}, Peter Brune\inst{1}, Matt Knepley\inst{3}, Dave May\inst{4}, Barry Smith\inst{1}}


% - Use the \inst command only if there are several affiliations.
% - Keep it simple, no one is interested in your street address.
\institute
{
  \inst{1}{Mathematics and Computer Science Division, Argonne National Laboratory} \\
  \inst{2}{Columbia University} \\
  \inst{3}{Computation Institute, University of Chicago} \\
  \inst{4}{ETH Z\"urich}
}

\date{2012-02-06}

% This is only inserted into the PDF information catalog. Can be left
% out.
\subject{Talks}


% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

% \pgfdeclareimage[height=0.5cm]{university-logo}{university-logo-filename}
% \logo{\pgfuseimage{university-logo}}



% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
% \AtBeginSubsection[]
% {
% \begin{frame}<beamer>
%   \frametitle{Outline}
%   \tableofcontents[currentsection,currentsubsection]
% \end{frame}
% }

\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection]
  \end{frame}
}

% If you wish to uncover everything in a step-wise fashion, uncomment
% the following command:

% \beamerdefaultoverlayspecification{<+->}

\begin{document}
\lstset{language=C}
\normalem

\begin{frame}
  \titlepage
\end{frame}

\section{Multiphysics and methods}
\input{slides/MultiphysicsExamples.tex}
\input{slides/MonolithicOrSplit.tex}
\input{slides/FieldSplit.tex}

\input{slides/VHTSolversNesting.tex}
\begin{frame}[fragile]{Example $3\times 3$ problem with nested $2\times 2$ split}
\begin{Verbatim}[formatcom=\footnotesize]
-fieldsplit_s_ksp_type gcr
-fieldsplit_s_ksp_rtol 1e-1
-fieldsplit_s_ksp_monitor_vht
-fieldsplit_s_ksp_monitor_singular_value
-fieldsplit_s_pc_type fieldsplit
-fieldsplit_s_pc_fieldsplit_type schur
-fieldsplit_s_pc_fieldsplit_real_diagonal
-fieldsplit_s_pc_fieldsplit_schur_factorization_type lower
-fieldsplit_s_fieldsplit_u_ksp_type gmres
-fieldsplit_s_fieldsplit_u_ksp_max_it 10
-fieldsplit_s_fieldsplit_u_pc_type asm
-fieldsplit_s_fieldsplit_u_sub_pc_type ilu
-fieldsplit_s_fieldsplit_u_sub_pc_factor_levels 1
-fieldsplit_s_fieldsplit_u_ksp_converged_reason
-fieldsplit_s_fieldsplit_p_ksp_type preonly
-fieldsplit_s_fieldsplit_p_ksp_max_it 1
-fieldsplit_s_fieldsplit_p_pc_type jacobi
-fieldsplit_e_ksp_type gmres
-fieldsplit_e_ksp_converged_reason
-fieldsplit_e_pc_type asm
-fieldsplit_e_sub_pc_type ilu
-fieldsplit_e_sub_pc_factor_levels 2
\end{Verbatim}
\end{frame}

\begin{frame}[fragile]{Monolithic approaches}
  \begin{block}{Parallel direct solver}
    \begin{Verbatim}[formatcom=\footnotesize]
      -dm_mat_type aij -pc_type lu -pc_factor_mat_solver_package mumps
    \end{Verbatim}
  \end{block}
  \begin{block}{Coupled nonlinear multigrid accelerated by NGMRES with multi-stage smoothers}
  \begin{Verbatim}[formatcom=\footnotesize]
    -lidvelocity 200 -grashof 1e4
    -snes_grid_sequence 5 -snes_monitor -snes_view
    -snes_type ngmres
    -npc_snes_type fas
    -npc_snes_max_it 1
    -npc_fas_coarse_snes_type ls
    -npc_fas_coarse_ksp_type preonly
    -npc_fas_snes_type ms
    -npc_fas_snes_max_it 1
    -npc_fas_ksp_type preonly
    -npc_fas_pc_type pbjacobi
    -npc_fas_snes_ms_type m62
    -npc_fas_snes_max_it 1
  \end{Verbatim}
\end{block}
\end{frame}

\section{Coupling software in PETSc}
\input{slides/PETSc/Coupling.tex}
\input{slides/PETSc/LocalSpaces.tex}
\input{slides/PETSc/MatGetLocalSubMatrix.tex}
\setbeamertemplate{background canvas}{}
\includepdf[pages=1-3]{davemay.pdf}

\section{Hardware and consequences}
\begin{frame}{On-node hardware roadmap}
  \begin{block}{Hardware trends}
    \begin{itemize}
    \item More cores (keep hearing $\bigO(1000)$ per node)
    \item Long vector registers (32B for AVX and BG/Q, 64B for MIC)
    \item Must use SMT to hide memory latency (POWER7)
    \item Must use SMT for floating point performance (GPU, BG/Q)
    \item Large penalty for non-contiguous memory access
    \end{itemize}
  \end{block}
  \begin{block}{``Free flops'', but how can we use them?}
    \begin{itemize}
    \item High order methods good: better accuracy per storage
    \item High order methods bad: work unit gets larger
    \item GPU threads have very little memory, must keep work unit small
    \item Need library composability, keep user contribution embarrassingly parallel
    \end{itemize}
  \end{block}
\end{frame}

\begin{frame}{How to program this beast?}
  \begin{itemize}
  \item Decouple physics from discretization
    \begin{itemize*}
    \item Expose small, embarrassingly parallel operations to user
    \item Library schedules user threads for reuse between kernels
    \item User provides physics in kernels run at each quadrature point
    \item Continuous weak form: find $u \in \VV_D$
      \[ v^T F(u) \sim \int_\Omega v \cdot {\color{green!70!black} f_0(u,\nabla u)}
      + \nabla v \tcolon {\color{green!70!black} f_1(u,\nabla u)} = 0, \qquad \forall v \in \VV_0 \]
    \item Similar form at faces, may involve Riemann solve
    \end{itemize*}
  \item Library manages reductions
    \begin{itemize*}
    \item Interpolation and differentiation on elements
    \item Exploit tensor product structure to keep working set small
    \item Assembly into solution/residual vector (sum over elements)
    \end{itemize*}
  \end{itemize}
\end{frame}

\input{slides/Dohp/TensorFEM.tex}
\input{slides/Dohp/RepresentationOfJacobians.tex}
\newcommand\smallterm[1]{{\color{gray} #1}}
\begin{frame}{Conservative (non-Boussinesq) two-phase ice flow}
  Find momentum density $\rho\uu$, pressure $p$, and total energy density $E$:
  \begin{gather*}
    (\rho\uu)_t + \div (\smallterm{\rho\uu\otimes\uu} - \eta D\uu_i + p\bm 1) - \rho \bm g = 0 \\
    \rho_t + \div \rho\uu = 0 \\
    E_t + \div \big((E+p)\uu - k_T\nabla T - k_\omega\nabla\omega \big) - \eta D\uu_i\tcolon D\uu_i - \smallterm{\rho\uu\cdot\bm g} = 0
  \end{gather*}
\begin{itemize}
\item Solve for density $\rho$, ice velocity $\uu_i$, temperature $T$, and melt fraction $\omega$ using constitutive relations.
  \begin{itemize}
  \item Simplified constitutive relations can be solved explicitly.
  \item Temperature, moisture, and strain-rate dependent rheology $\eta$.
  \item High order FEM, typically $Q_3$ momentum \& energy
  \end{itemize}
\item DAEs solved implicitly after semidiscretizing in space.
\item Preconditioning using nested fieldsplit
\end{itemize}
\end{frame}
\input{slides/Dohp/TensorVsAssembly.tex}

\input{slides/HardwareArithmeticIntensity.tex}

\begin{frame}{Prospects for reducing synchronization}
  \begin{itemize}
  \item Dot products and norms
    \begin{itemize}
    \item orthogonality is a powerful concept
    \item dot product/norm fusion in CG variants
    \item latency-tolerant Krylov methods, TSQR for GMRES
    \item nonlinear methods (e.g. NGMRES, BFGS, line searches)
    \item hierarchical methods to limit system-wide norms
    \item setting up smoothers and coarsening rates for AMG
    \end{itemize}
  \item additive coarse grids
  \item subphysics on subcommunicators, even within multigrid context
  \item $s$-step methods (and other fusion)
    \begin{itemize}
    \item often spoiled by algorithmic requirements of preconditioning
    \item relevant for multigrid smoothers
    \item difficult crossovers for 3D problems
    \end{itemize}
  \end{itemize}
\end{frame}

% \begin{frame}{$S$-step methods in 3D}
%   \includegraphics[width=1.1\textwidth]{figures/SStep.pdf}
% \end{frame}

\begin{frame}{Multigrid is \emph{always} strong scaling}
  \begin{itemize}
  \item Finest level is chosen by the application (might have big subdomains)
  \item All coarsened levels choose communicator size based on strong scaling limit
  \item Optimizing the strong scaling limit pays off consistently
  \item Rapid coarsening is important (2:1 semi-coarsening not okay any more)
  \end{itemize}
\end{frame}

\begin{frame}{Advanced Analysis}
  \begin{itemize}
  \item Uncertainty quantification
    \begin{itemize}
    \item intrusive vs unintrusive methods, multilevel
    \item uncertainty in modeling error
    \item use subdifferentials for non-smooth processes
    \item unified handling of heterogeneous observational data
    \end{itemize}
  \item PDE-constrained optimization
    \begin{itemize}
    \item multi-objective
    \item robustness
    \item rich problem description
    \item fusing algorithmic steps (LNK and coupled DD fuse gradients with progress)
    \end{itemize}
  \item Exploring stability manifolds
    \begin{itemize}
    \item solving bordered linear and nonlinear systems
    \end{itemize}
  \item nearly time-periodic nonlinear problems
    \begin{itemize}
    \item identifying cycles in ocean models, turbomachinery
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Software challenges}
  \begin{itemize}
  \item Which interfaces do users have to interact with?
    \begin{itemize}
    \item ``F''ramework vs library
    \item Extensibility is critical for multiphysics
    \end{itemize}
  \item Asynchronous interfaces crossing module boundaries
    \begin{itemize}
    \item How to ensure progress?
    \end{itemize}
  \item Merge communication on multiple levels or between multiple physics
  \item Fusing coarse level operations
  \item Working with non-nested communicators is tricky
  \item Current solutions for hierarchical memory are bad for libraries
    \begin{itemize}
    \item I want a communicator-like object
    \item I want a way to allocate memory explicitly/relative to algorithmic dependencies instead of implicit ``first touch''
    \end{itemize}
  \item Time integration: IMEX, multirate, parallel in time
    \begin{itemize}
    \item method of lines: $g(\dot u,u,t) = f(u,t)$
    \item Lax-Wendroff time integration is harder for composable software
    \end{itemize}
  \end{itemize}
\end{frame}

\end{document}