
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Welcome}

Two of the most exciting areas of ML, combining forces.
\pause
\begin{columns}
\begin{column}{0.7\textwidth}
\begin{itemize}
\item Probabilistic models $p_\theta(x) = \int p_\theta(x|z)p_\theta(z)dz$
\begin{itemize}
\item interpretable structure, quantified uncertainty,...
\end{itemize}
%\item Deep neural networks $f_\theta(\cdot) = f_\theta^{(\ell)}\left(...\left(f_\theta^{(0)}(\cdot)\right)\right)$: 
\pause
\item Deep neural networks $f_\theta(\cdot) = f_\theta^{(\ell)}\circ... \circ f_\theta^{(0)}(\cdot)$
\begin{itemize}
\item expressivity, simple learnability,...
\end{itemize}
\end{itemize}
\vfill
\pause
\begin{itemize}
 \item \emph{Deep Generative Model} (DGM)
 \begin{itemize}
 \item $p_\theta(x)$: latent $p_\theta(z)$, (noisy) map $f_\theta: \mathcal{Z} \rightarrow \mathcal{X}$.
% \item We will chop up this definition in many ways, but... 
\end{itemize}
\pause
\item Central theme: 
\graybox{Use probabilistic models where we have inductive bias; \\ Use flexible function approximators where we do not.}

\vfill \item We will explore this theme with VAE, GAN, and more.
\end{itemize}

\begin{column}{0.05\textwidth}
\end{column}

\end{column}
\begin{column}{0.25\textwidth}
\begin{center}
\includegraphics[height=2.6cm]{fig/vae0}
\end{center}
\vfill
\begin{center}
\includegraphics[height=3.4cm]{fig/gan3}
\end{center}
\end{column}
\end{columns}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Criteria}

Prerequisites
\begin{columns}
\begin{column}{0.7\textwidth}
\begin{itemize}
\item Probabilistic machine learning $\supseteq$ \href{http://www.cs.columbia.edu/~blei/fogm/2018F/index.html}{Blei's PGM}
\vfill
\item Deep learning $\supseteq$ \href{http://stat.columbia.edu/~cunningham/teaching/slides_AML2.pdf}{Cunningham's AML}
\vfill
\item Programming $\supseteq$ \{python, tensorflow, pytorch, keras\}
\end{itemize}
\end{column}
\begin{column}{0.25\textwidth}
\begin{center}
\includegraphics[width=\textwidth]{fig/python}
\end{center}
\begin{center}
\includegraphics[width=0.25\textwidth]{fig/tf} ~~~~
\includegraphics[width=0.25\textwidth]{fig/torch}
\end{center}
\end{column}
\end{columns}
\vfill \pause
Attitude
\begin{itemize}
\item Excitement about this research area
\item Desire to write a paper / work on a project in this domain
\item Game to be lively and involved in seminar
\end{itemize}
\vfill \pause
Also
\begin{itemize}
\item Alignment with your research is terrific
\item Diverse and nontraditional students welcome
\end{itemize}
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Then again, maybe not}

You aren't doing ML research already
\begin{itemize}
\item This course will be fast paced, Ph.D. level; prerequisites are real
\end{itemize}
\vfill
You are just looking for course credit 
\begin{itemize}
\item The format depends on the involvement and enthusiasm of a dedicated group
\end{itemize}
\vfill
You are fully committed with an unrelated project
\begin{itemize}
\item This course will be a lot of work: reading, participation, lecturing, projects
\end{itemize}
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Format}

Tone
\begin{itemize}
\item fun and fast paced
\item technical and creative
\item positive and inclusive
\end{itemize}
\vfill
Seminar
\begin{itemize}
\item Journal club-style: everybody reads, everybody participates
\item RealDeal(TM): no phones, no email, no text, etc; tablets ok
\end{itemize}
\vfill
Presentation 
\begin{itemize}
\item You prepare and lead the seminar (slides, whiteboard, etc.)
\item Pre-meetings with me to take me through it (Mondays 1000 ok?)
\end{itemize}
\vfill
Project...
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Project}

Key points
\begin{itemize}
\item novel contribution: modeling, application, theory, code, etc.  
\item meaningful software implementation of a DGM
\item project report with content and style of a top machine learning conference 
\begin{itemize}
\item intro, lit review, model, implementation, results, figures, discussion,...
\end{itemize}
\item given: ideas, templates (paper, repo), support (Wednesdays 1600 ok?), etc. 
\end{itemize}
\vfill \pause
Last time (STAT G8325, 2015-2016)
\begin{itemize}
\item ML modeling papers published: \hfill {\tiny [Fagan$+$ UAI 2016, Bloem-Reddy$+$ ICML 2016, Fagan$+$ AISTATS 2017]  }
\item Applications papers: \hfill {\tiny [Elsayed$+$ Nature Communications 2016, Mena$+$ PLOS CB 2017]}
\item Code: new feature set added to {\tt STAN}
\end{itemize}
\vfill \pause
In-bounds topics
\begin{itemize}
\item complicated probability distribution you want to learn
\item intractable probabilistic inference where MCMC or similar might not do
\item learn latent structure in high-dimensional, noisy data
\item learn to sample from a complex and high-d $p_\theta(x)$
\item applied DGM research (materials? drug discovery? quant trading?)
\item questions?
\end{itemize}
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Topics}

{\scriptsize
\begin{center}
  \begin{tabular}{cll}
    \toprule
    Part & Week Counter & Content \\
    \midrule
    I & $1$  &  Introduction and basics \\
       && \tabitem Reading: {\bf \cite{kingma2013auto, goodfellow2014generative}}; \cite{rezende2014stochastic} \\
%
    II & $+1$ & Structured VAE models\\
       && \tabitem Reading: {\bf \cite{Archer:2015aa, johnson2016composing}}; \cite{gao2016linear, krishnan2015deep} \\
 %
   III & $+2-3$ & Conditional and discrete DGMs \\
       && \tabitem Reading:  { TBD} \\
   %IV & $+1$ & Guest lecture on discrete neural process VAE \\
    %   && \tabitem Reading:  {\bf \cite{pakman2019, pakman2019b} \\
   IV & $+2-3$ & Disentangling, interpreting, and geometry of VAE latents  \\
       && \tabitem Reading:  {TBD} \\
     - & $+1$ & (spring holiday) \\
      && \\
   V & $+2-3$ & VAE next-level mechanics: inference, reparameterizations, lower bounds, etc.  \\
       && \tabitem Reading:  {TBD} \\
   VI & $+1$ & VAE $\rightarrow$ GAN: learning likelihood free models \\
   	    && \tabitem Reading:  {\bf \cite{mohamed2016learning}}; \cite{tran2017hierarchical, kusner2016gans} \\
   VI & $+1$ & AR density estimation: NADE, MADE, NVP, norm flow, PixelRNN, WaveNet, etc.  \\
       && \tabitem Reading:  {TBD} \\       
   VII & $+2-3$ & GAN mechanics: basic issues, $f$, Wasserstein, OT,  etc.  \\
       && \tabitem Reading:  { TBD} \\
   VIII & $+1-3$ & GAN applications: superresolution, CycleGAN, steganography, etc.  \\
       && \tabitem Reading:  { TBD} \\
   IX & $+1-2$ & GAN issues and emerging theory  \\
       && \tabitem Reading:  {TBD} \\
    X  &&  Project final reports \\
    \bottomrule
  \end{tabular}
\end{center}
}
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Great, but I'm not registered...}

\pause 
Course enrollment will be prioritized roughly as:
\begin{enumerate}
\item Columbia Statistics PhD students
\item other Columbia students
\item Columbia auditors
\item non-Columbia enthusiasts
\item ...
\item faculty
\end{enumerate}

\vfill \pause

Homework (Due Friday 25 January at 4:59pm)
\graybox{ \centering{\tt https://goo.gl/forms/Y9A1DW8s81ZaNKs83}}

\vfill \pause

Despite (because of?) all of these particulars, I do hope you will join...

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\sectionframe{At last, content}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Modeling}

A (the?) central problem in statistics is fitting a \emph{model} to \emph{data}:

\graybox{
\centering $\mathcal{M} = \left\{ p_\theta : \theta \in \Theta \right\}~~,~~ X = \{x_1,...,x_n\} ~~\rightarrow~~ \arg\max_\theta \log p_\theta(X)$
}

\vfill

Excellent strategy: partition randomness and structure...
\begin{itemize}
\item Generate \emph{latent} $z_i \sim p_0(z)$, compute $x_i = f_\theta(z_i)$ with (noisy) $f_\theta$
\item Induces a (possibly) complex model/family of distributions $p_\theta(x)$
\item This approach can take a few forms...
\end{itemize}

\vfill

\emph{Prescribed} probabilistic models:
\begin{itemize}
\item form $p_\theta(x)$ directly, proceed to inference, parameter estimation, regularization, etc.
\item many simple models have this form...
\end{itemize}

\vfill


\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Implicit Probability Models}


\emph{Implicit} probabilistic models:
\begin{itemize}
\item Specify a latent $p(z)$ followed by a procedure $f_\theta: \mathcal{Z} \rightarrow \mathcal{X}$
\item Key point: in this setting, sampling data is almost always easy
\item Sometimes the whole problem is easy: remember inversion sampling?
$$ z \sim Unif(0,1)~~~~ x = F^{-1}_\phi(z)~~~ \rightarrow~~~ x \sim Exp(\phi)$$
{\hfill \scriptsize $F_\phi$ is the cdf of the exponential distribution, $F_\phi(x) = 1 - \exp(-\phi x)$, with $F^{-1}_\phi (z) = -\phi \log (1-z)$}
\item More often $f_\theta$ induces an intractable log likelihood:
$$ p_\theta(x)= \frac{\partial}{\partial x_1}...\frac{\partial}{\partial x_d} \int_{\left\{ f_\theta(z)\leq x \right\}} p_\theta(z) dz $$
\end{itemize}

\begin{enumerate}
\item Hard: tractable, specified conditional likelihood (bayesian/graphical models):
$$ p_\theta(x)= \int p_\theta(x|z) p_\theta(z) dz $$ 
\item Harder: stochastic sampling procedures (diff eq, ecology, weather, finance, etc):
$$  p_\theta(x|z) = ?? $$ 
\end{enumerate}

 \vfill
 
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{IPM with deep neural networks}

With almost no exception, hereafter DGM $==$ IPM.  The main idea:

\begin{itemize}
\item Sample randomness from a particularly easy distribution $z \sim \mathcal{N}(0,I)$
\item Use a deep neural network as the structure map $f_\theta$
\item Best of both worlds? ...flexible, expressive $p_\theta(x)$ that is easy to sample and learn
\end{itemize}

\vfill

1. Tractable $p_\theta(x|z)$ $\rightarrow$ EM, VI, ELBO,...

\begin{center}
\includegraphics[width=1.4cm]{fig/fig_bishop_KL_bimodal_2.pdf}
\includegraphics[width=1.4cm]{fig/fig_bishop_KL_bimodal_3.pdf}
\end{center}

\begin{itemize}

\item Today: the \emph{variational autoencoder} \cite{kingma2013auto}
\end{itemize}

\vfill 
2. Intractable $p_\theta(x|z)$ $\rightarrow$ generative modeling...
\\
\begin{center}
$ z_i \sim \mathcal{N}(0,I) ~~~~ \rightarrow ~~~~ g_\theta(z_i) ~~~~ \rightarrow~~~~ $ \inlinegraphics{fig/mnist_digit_post}{1.5}{-0.7}
\end{center}

\begin{itemize}

\item Today: the \emph{generative adversarial network} \cite{goodfellow2014generative}
\end{itemize}
\vfill
3. Much more to come on IPM approaches (see Part VI and \cite{mohamed2016learning}).

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%
% now moving to VAE
%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: EM}

We wish to maximize $\log p_\theta(x) = \log \int p_\theta(x|z)p_\theta(z) dz$.  EM:
\begin{eqnarray*}
\log p_\theta(x) & = & \log \int p_\theta(x|z)p_\theta(z) dz \\ 
& = & \log \int p_\theta(x|z)p_\theta(z) \frac{q(z)}{q(z)} dz \\
& \geq & \int q(z) \log \frac{ p_\theta(x|z)p_\theta(z) }{q(z)} ~~~~~~~ = ~~~ \mathcal{L}(q,\theta)
\end{eqnarray*}

Key observation: if $q$ is the posterior $p_\theta(z|x)$, then bound is achieved.
\begin{center}
\includegraphics[width=0.7\textwidth]{fig/em}\\
\end{center}
{\hfill \scriptsize [Turner and Sahani 2011]}
\vfill
NB: A is $\mathcal{L}(\theta,q)$ (ignore right panel); B is EM as a function of $\theta$ (M steps).

\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: (sidebar) MM algorithms}

EM is a special case of a \emph{minorize-maximize} algorithm (MM):

\begin{itemize}
\item Bound $g(\theta)$ has to be tight.  Or at least gap independent of $\theta$.
\item Specifically two conditions (here majorize-minimize):
\begin{eqnarray*}
g\left(\theta | \theta^m \right) &\geq & f(\theta) ~~~~ \forall \theta \\
g\left(\theta^m | \theta^m \right) &=& f(\theta^m)
\end{eqnarray*}

\begin{center}
\includegraphics[width=0.8\textwidth]{fig/mm}\\
\end{center}
{\hfill \scriptsize [Hunter and Lange 2004]}

\vfill
NB: here $f(\theta) = \log p_\theta(x)$, aka the ELBO is tight when $q$ is posterior.
\vfill

\end{itemize}
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: variational posterior}

Suppose we can't calculate the posterior; then \emph{for a fixed $\theta$} we can still:

\begin{eqnarray*}
\arg\min_\phi KL\left(q_\phi(z) || p_\theta (z|x)\right) & = & \arg\max_\phi  \mathcal{L}(\phi,\theta) 
\end{eqnarray*}

Notice
\begin{itemize}
\vfill\item We call this maximizing the ELBO with a model $\mathcal{Q} = \left\{ q_\phi(z) : \phi \in \Phi\right\} $
\vfill\item It's VI, and is also an ok thing to do. 
\vfill\item Lots of textbook work here, but...

{\hfill \scriptsize [Murphy 2014, Mackay 2003, Bishop 2006, Wainright and Jordan 2008, Jordan et al 1999, etc.]}

\end{itemize}

\graybox{ \emph{There is as much ``art'' as there is ``science'' in our current understanding of how variational methods can be applied to probabilistic inference.} \\ {\footnotesize -- [Jordan, Ghahramani, Jaakkola, Saul 1999] }}

Warning: we will get confused about our goal -- $\arg\max \log p_\theta(x)$ or $\arg\min KL(q ||p)$...
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: vEM (these days $\approx$ VI)}

\begin{itemize}
\item Why not do both -- variational EM (replace E step with VI): 
\end{itemize}
%
\begin{eqnarray*}
\text{vE-step} & & \\
\arg\min_\phi KL\left(q_\phi(z) || p_\theta (z|x)\right) & = & \arg\max_\phi  \mathcal{L}(\phi,\theta) \\
\text{M-step} & & \\
 \arg\max_\theta  \mathcal{L}(\phi,\theta) &&
\end{eqnarray*}


\begin{itemize}
\item An old and well practiced idea [Jordan et al 1999]
\item Extremely common now [Kingma and Welling 2013, ...]
\item Mechanics:
    \begin{equation*}
      \begin{split}
      KL(q_\phi(z) |p_\theta(z|x))
      =&\ 
      E_{q_\phi} \left( \log\frac{q_\phi(z)}{p_\theta(z|x)}\right)\\
      =&\ 
     E\left(\log q_\phi(z)\right)
      -
      E\left(\log p_\theta(z|x)\right)\\
      =&\ 
      E\left(\log q_\phi(z)\right)
      -
      E\left(\log p_\theta(z,x)\right)
      +
      \log p_\theta(x) \\
      \propto &\ 
      E_{q_\phi}\left( \log q_\phi(z)\right)-E_{q_\phi} \left(\log p_\theta(z,x)\right) \\
      = &\ \mathcal{L}(\phi,\theta)
      \end{split}
    \end{equation*}
\item Is this ok?
\end{itemize}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: (sidebar) ELBO gap}

Notice
\begin{itemize}
\item ...we are violating the second condition of an MM algorithm
\item ...we are changing the optimization landscape each iteration  
\item A key result from [Turner and Sahani 2011]
\end{itemize}

\begin{center}
\includegraphics[width=0.8\textwidth]{fig/vem}\\
\end{center}


Reminder
\begin{itemize}
\item not an issue with VI 
\item not an issue with EM.  
\item Rather the (careless?) combination of the two.
\item Why is EM ok in the first place (uneven: posterior $+$ global MLE...)?
\item Is this a bigger or smaller issue in neural network land? 
\end{itemize}
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: (sidebar) ELBO gap}

Simple closed-form example that will allow us to explore this pathology:

\begin{eqnarray*}
p_\alpha (z) & = & \mathcal{N}\left( 0 , \begin{bmatrix} 1 & \alpha \\ \alpha & 1 \end{bmatrix} \right) \\
p(x_i|z_i) & = & \mathcal{N}\left( z_i , \begin{bmatrix} \beta & 0 \\ 0 & \beta \end{bmatrix} \right) \\ 
q_{\mu,\sigma}(z) & = & \prod_{d=1}^2 \mathcal{N}\left(\mu^d,\sigma^d\right)  
\end{eqnarray*}

Note: 
\begin{itemize}
\item Evidence $p_\theta(x)$ and posterior $p_\theta(z|x)$ are closed-form
\item $\theta = \alpha$ for simplicity
\item $\phi = \left\{ \mu^1, \mu^2, \sigma^1 , \sigma^2 \right\}$
\item Variational parameters $\arg\max_{\phi} \mathcal{L}(\phi,\theta)$ also closed form...
\end{itemize}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: (sidebar) ELBO gap}

\vspace{-0.3cm}
\begin{center}
\includegraphics[width=0.6\textwidth]{fig/example2}\\
\end{center}
\vspace{-0.6cm}
\begin{center}
\includegraphics[width=0.7\textwidth]{fig/example3}
\end{center}

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: (sidebar) ELBO gap}

We dwell on this for a few reasons:

\begin{itemize}
\vfill \item vEM (VI) is a biased estimator of $\theta$
\vfill\item ELBOs are lower bounds and thus can not be compared across models
\vfill\item Other issues with VAE will arise later (Parts IV-V), but this is different
\vfill\item Remember why we even do EM, VI, etc (literature is confused on this point)
\end{itemize}
\vfill

\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

 
 % to return...



%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: variational autoencoder}

Armed with vEM, we: 
    \begin{itemize}
\item choose a generative model $\mathcal{M} = \{ p_\theta(x|z) p_\theta(z) : \theta \in \Theta\}$
\item  choose a variational model $\mathcal{Q} = \{q_\phi(z|x): \phi \in \Phi\}$
\item maximize:
    \begin{equation*}           
     \mathcal{L}(\phi, \theta) =  - E_{q_\phi}\left( \log q_\phi(z|x)\right) + E_{q_\phi} \left(\log p_\theta(z,x)\right)
    \end{equation*}
    \end{itemize}

\vfill
View this setup as \emph{dimension reduction}:
\begin{itemize}
\item $p_\theta(x|z)$ is a \emph{probabilistic decoder}, converting latent code $z$ to observed data $x$
\item $q_\phi(z|x)$ is a \emph{probabilistic encoder}, converting observed data $x$ to  latent code $z$ 
\end{itemize}

\vfill

We need only choose our model particulars...


\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Tractable $p_\theta(x|z)$: variational autoencoder}

\begin{columns}
\begin{column}{0.7\textwidth}
ELBO:
\begin{equation*}           
     \mathcal{L}(\phi, \theta) =  - E_{q_\phi}\left( \log q_\phi(z|x)\right) + E_{q_\phi} \left(\log p_\theta(z,x)\right)
    \end{equation*}
        %\vspace{0.5cm}
    \begin{itemize}
        \item Suppose $z \in \mathbb{R}^d$ and $x \in \mathcal{X}$
    \end{itemize} 
\end{column}
\begin{column}{0.3\textwidth}
\begin{center}
\includegraphics[width=2.4cm]{fig/vae0}
\end{center}
\end{column}
\end{columns}
\vfill

DGM: use deep networks as flexible, expressive function families:
$$q_\phi( z | x) = \mathcal{N}\left( \mu_\phi( x) , \sigma^2_\phi(x) \right)$$
\begin{itemize}
\item Here both $\mu_\phi$ and $\sigma_\phi$ are MLPs that map $\mathcal{X} \rightarrow \mathbb{R}^{d}$
\item Perhaps easier to view this from the perspective of a noise variable $\epsilon$:
$$ \epsilon \sim \mathcal{N}(0,I_d) ~~~~\text{and}~~~~ z = \mu_\phi(x) + \sigma_\phi(x) \odot \epsilon ~~~~\rightarrow~~~~  z|x \sim \mathcal{N}\left( \mu_\phi( x) , \sigma^2_\phi(x) \right)$$
\item This \emph{reparameterization trick} makes it simple to sample from $q_\phi(z|x)$ (Part V)
\item Note: this gaussian is one basic choice, but many others are used (Parts II-V)
\item Note: full VAE also has DGM for $p_\theta$ (but needn't)
\end{itemize}

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Tractable $p_\theta(x|z)$: stochastic optimization}

We still have the issue of calculating (and differentiating!) these expectations:
\begin{equation*}      
     \arg\max_{\phi,\theta} \mathcal{L}(\phi, \theta) =  \arg\max_{\phi,\theta} - E_{q_\phi}\left( \log q_\phi(z|x)\right) + E_{q_\phi} \left(\log p_\theta(z,x)\right)
    \end{equation*}
  
  \vfill
Turn to stochastic optimization and mini-batch gradient descent: 
\begin{itemize}
\item Draw a noise minibatch $\epsilon_1,...,\epsilon_L$ iid from $\mathcal{N}(0,I)$ (often $L=1$)
\item Draw a data minibatch $x_1,...,x_M$ from the dataset (often $M=100$ or similar)
\item Compute $z_{m\ell} = \mu_\phi(x_m) + \sigma_\phi(x_m)\odot \epsilon_\ell$ (...sidesteps explicit diff of $E_{q_\phi}(\cdot)$!)
\item Optimize with SGD (Adam, etc.) as usual with stochastic gradients:
$$ \nabla_{\phi,\theta} \mathcal{L}(\phi,\theta) \approx  \frac{N}{ML} \sum_{\ell=1}^L\sum_{m=1}^M -\nabla_\phi \log q_\phi(z_{m\ell}|x_m) +  \nabla_{\phi,\theta} \log p_\theta(z_{m\ell},x_m) $$
%$$ \hat{\mathcal{L}}(\phi) =  - \frac{1}{M} \sum_{m=1}^M \log q_\phi(z_m|x_m) + \frac{1}{M} \sum_{m=1}^M \log p_\theta(z_m,x_m) $$
\item note that in some cases entropy (first) term will have closed form $E_{q_\phi}$...
\end{itemize}

\vfill

Optimizing this objective:
\begin{itemize}
\item Learns a generative distribution $p_\theta(x)$
\item Learns an \emph{amortized} approximation $q_\phi(z| x)$ that can be queried for any $x$ 
\item We call this general approach variational autoencoding (VAE) \cite{kingma2013auto}
\end{itemize}


\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Tractable $p_\theta(x|z)$: VAE in action}

Learn the autoencoder and then:
\begin{itemize}
\item Choose a point $z_i$ in latent space (not drawing from the posterior!)
\item Decode this point with $x_i \sim p_\theta(x_i | z_i)$:
\end{itemize}
\begin{center}
\includegraphics[height=5.8cm]{fig/vae2}~~~~~
\includegraphics[height=5.8cm]{fig/vae3}
\end{center}

Learns a manifold of simple images and how to generate...

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{From VAE to GAN}


\begin{columns}
\begin{column}{0.7\textwidth}
Variational autoencoders:
\begin{itemize}
\item ...are usually thought of as doing inference (but...) 
\item ...are seen as dimension reduction / latent modeling
\item can generate, but that is not their specific design...
\end{itemize}

\end{column}
\begin{column}{0.3\textwidth}
\begin{center}
\includegraphics[width=2.4cm]{fig/vae0}
\end{center}
\end{column}
\end{columns}
\vfill


A useful minimax analogy for directly solving the data generation problem: 

\begin{center}
\includegraphics[width=10.4cm]{fig/gan10}
\end{center}
{\hfill \scriptsize \tt https://www.secretservice.gov/data/KnowYourMoney.pdf}


\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Generative adversarial networks}

From a deep learning perspective:
\begin{itemize}
\item True data samples $x_i^{D} \sim p_{data}(x)$... (minibatch) draws from the training set
\item The latent code $z_i \sim \mathcal{N}(0,I)$
\item The \emph{generator} neural network $x_i^G = G_{\theta_G}(z_i)$ 
\item The \emph{discriminator} neural network $D_{\theta_D} (x_i) \rightarrow [0,1]$
\end{itemize}

\begin{center}
\includegraphics[width=8.4cm]{fig/gan2}
\end{center}
\vspace{-0.4cm}
{\hfill \tt \tiny [image from http://cognitivechaos.com/understanding-generative-adversarial-networks/]}
\vfill

\begin{itemize}
\item The discriminator classifies fake vs real images
\item The generator adapts to fool the discriminator
\item This two-player game is repeated...
\end{itemize}

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Generative adversarial networks}

\begin{center}
\includegraphics[width=8.4cm]{fig/gan2}
\end{center}

Specify the following objective:
\begin{equation*}
\min_{\theta_G} \max_{\theta_D}~~\left[ E_{x \sim p_{data}} \left( \log D_{\theta_D}(x) \right) + E_{z \sim p(z)} \left( \log \left( 1 - D_{\theta_D}\left( G_{\theta_G}( z) \right) \right) \right) \right]
\end{equation*}


\begin{itemize}
\item $D_{\theta_d}(x_i)$ gives the probability ($\in [0,1]$) that $x^D_i$ is genuine (from data distribution)
\item $1 - D_{\theta_D}\left( G_{\theta_G}(z_i) \right) $ gives the probability that $x^G_i$ is \emph{fake}
\item $\min_{\theta_G}$ attempts to minimize the probability of being caught as a fake
\item $\max_{\theta_D}$ attempts to maximize discriminability (reals $\uparrow$, fakes $\downarrow$)...
\end{itemize}

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Generative adversarial networks}

\begin{equation*}
\min_{\theta_G} \max_{\theta_D}~~\left[ E_{x \sim p_{data}} \left( \log D_{\theta_D}(x) \right) + E_{z \sim p(z)} \left( \log \left( 1 - D_{\theta_D}\left( G_{\theta_G}( z) \right) \right) \right) \right]
\end{equation*}

\begin{center}
\includegraphics[width=10.4cm]{fig/gan4}
\end{center}

Here:
\begin{itemize}
\item discriminator $D(x)$ (blue); generative $p_{G}(x)$ (green); true $p_{data}(x)$ (black)
\item if arbitrarily expressive (2nd panel), $\max_D$ optimizes to $D_{\theta_D}(x) = \frac{p_{data}(x)}{p_{data}(x) + p_{G}(x)}$
\item If everything works, eventually $p_{G}$ is indistinguishable from $p_{data}$...
\end{itemize}

Note: 
\begin{itemize}
\item do not take this objective/optima as absolute truth (Parts VI-IX)
\item theory is starting to appear (Part IX)
\item a taste: \emph{mode collapse} and learning/generating the training set
\end{itemize}

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{GAN in action}


\begin{center}
\includegraphics[width=5.1cm]{fig/gan5}~
\includegraphics[width=5.1cm]{fig/gan6}\\
\includegraphics[width=5.1cm]{fig/gan7}~
\includegraphics[width=5.1cm]{fig/gan8}
\end{center}
{\hfill \tiny (right column images are nearest neighbor training points)}

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{GAN in action}


\begin{center}
\includegraphics[width=11.1cm]{fig/gan0}\\
\end{center}

Is this good?  What could we do with it if it were? (Part VIII) 
\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{GAN in action}


\begin{center}
\includegraphics[width=8.1cm]{fig/gan1}
\end{center}

\vfill
\end{frame}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%




