#LyX 2.0 created this file. For more info see http://www.lyx.org/
\lyxformat 413
\begin_document
\begin_header
\textclass article
\use_default_options true
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman default
\font_sans default
\font_typewriter default
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100
\font_tt_scale 100

\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize default
\use_hyperref false
\papersize default
\use_geometry false
\use_amsmath 1
\use_esint 1
\use_mhchem 1
\use_mathdots 1
\cite_engine basic
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\use_refstyle 1
\index Index
\shortcut idx
\color #008000
\end_index
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Standard
Derivation of the KL divergence between two multivariate normal distributions
 parameterized by mean 
\begin_inset Formula $\mu$
\end_inset

 and diagonal precision 
\begin_inset Formula $\beta$
\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\int Q(x)\log Q(x)/P(x)dx
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\int Q(x)\log Q(x)dx-\int Q(x)\log P(x)dx
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\mathbb{E}_{x\sim Q}[\log Q(x)-\log P(x)]
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\mathbb{E}_{x\sim Q}[-\frac{1}{2}\beta_{Q}^{T}(x-\mu_{Q})^{*2}-\log Z_{Q}+\frac{1}{2}\beta_{P}^{T}(x-\mu_{P})+\log Z_{P}]
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\mathbb{E}_{x\sim Q}[-\frac{1}{2}\beta_{Q}^{T}(x-\mu_{Q})^{*2}+\frac{1}{2}\beta_{P}^{T}(x-\mu_{P})^{*2}]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}<(x_{i}-\mu_{Pi})^{2}>-\beta_{Qi}<(x_{i}-\mu_{Qi})^{2}>\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}<x_{i}^{2}-2x_{i}\mu_{Pi}+\mu_{Pi}^{2}>-\beta_{Qi}<x_{i}^{2}-2x_{i}\mu_{Qi}+\mu_{Q_{i}}^{2}>\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}(<x_{i}^{2}>-2<x_{i}>\mu_{Pi}+\mu_{Pi}^{2})-\beta_{Qi}(<x_{i}^{2}>-2<x_{i}>\mu_{Qi}+\mu_{Q_{i}}^{2})\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}(<x_{i}^{2}>-2\mu_{Qi}\mu_{Pi}+\mu_{Pi}^{2})-\beta_{Qi}(<x_{i}^{2}>-2\mu_{Qi}^{2}+\mu_{Q_{i}}^{2})\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}(<x_{i}^{2}>-2\mu_{Qi}\mu_{Pi}+\mu_{Pi}^{2})-\beta_{Qi}(<x_{i}^{2}>-\mu_{Qi}^{2})\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
var(x)=<x^2>-(<x>)^2 so <x^2>=var(x)+(<x>)^2=1/beta+mu^2
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}(1/\beta_{Qi}+\mu_{Qi}^{2}-2\mu_{Qi}\mu_{Pi}+\mu_{Pi}^{2})-\beta_{Qi}(1/\beta_{Qi}+\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]+\log Z_{P}-\log Z_{Q}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]-\log(2\pi)^{-n/2}|\beta_{P}^{-1}|^{-1/2}+\log(2\pi)^{-n/2}|\beta_{Q}^{-1}|^{-1/2}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]-\log(2\pi)^{-n/2}-\log|\beta_{P}^{-1}|^{-1/2}+\log(2\pi)^{-n/2}+\log|\beta_{Q}^{-1}|^{-1/2}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]-\log|\beta_{P}^{-1}|^{-1/2}+\log|\beta_{Q}^{-1}|^{-1/2}
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]+\frac{1}{2}\log|\beta_{P}^{-1}|-\frac{1}{2}\log|\beta_{Q}^{-1}|
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})\right]-\frac{1}{2}\log|\beta_{P}|+\frac{1}{2}\log|\beta_{Q}|
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\beta_{Qi}(\mu_{Qi}^{2}-\mu_{Qi}^{2})-\log\beta_{pi}+\log\beta_{Qi}\right]
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\frac{1}{2}\sum_{i}\left[\beta_{Pi}/\beta_{Qi}+\beta_{Pi}(\mu_{Qi}-\mu_{Pi})^{2}-1-\log\beta_{pi}+\log\beta_{Qi}\right]
\]

\end_inset


\end_layout

\end_body
\end_document
