Homec4science
Diffusion epfl (master) Browse

R2653/teaching/COM418/lectures/02 - DSP review/vectors.tex
22149fcac34amaster

Last Change
Raw File

vectors.tex
View Options

\documentclass[aspectratio=169]{beamer}
%\documentclass[aspectratio=169,handout]{beamer}
\def\stylepath{../styles}
\usepackage{\stylepath/com418}
\usepackage{pst-3dplot}
%\setbeameroption{show only notes}\def\logoEPFL{}
\setbeameroption{show notes}
\begin{document}
\begin{frame} \frametitle{Let's talk about Vector Spaces...}
Some spaces should be very familiar:
\begin{itemize}
\item $\mathbb{R}^2, \mathbb{R}^3$: Euclidean space, geometry
\item $\mathbb{R}^N, \mathbb{C}^N$: linear algebra
\end{itemize}
\vspace{2em}
Others perhaps not so much...
\begin{itemize}
\item $\ell_2(\mathbb{Z})$: space of square-summable infinite sequences
\item $L_2([a,b])$: space of square-integrable \textit{functions} over an interval
\end{itemize}
\vspace{1em}
\centering
\textbf{yes, vectors can be functions!}
\end{frame}
\begin{frame} \frametitle{Why use vector spaces in DSP?}
Easier math and unified framework for signal processing:
\begin{itemize}
\item same formalism for different types of signals (discrete-time, continuous-time, periodic, etc.)
\item bring geometric intuition to signal operations (e.g. orthogonality)
\item easy explanation of the Fourier Transform
\item easy explanation of sampling and interpolation
\item useful in approximation and compression
\end{itemize}
\end{frame}
\begin{frame} \frametitle{Why use vector spaces in DSP?}
The key characters in our story
\begin{itemize}
\item $\mathbb{C}^{N}$: space of finite-length (or periodic) discrete-time signals; equivalent to a Python array
\item $\ell_2(\mathbb{Z})$: space of infinite-length, finite-energy discrete-time signals; good for proofs
\item $L_2([-\pi, \pi])$: space of $2\pi$-periodic square-integrable complex functions of a real variable; this is where discrete-time Fourier transforms live
\item $L_2(\mathbb{R})$: space of finite-energy continuous-time signals; idealized analog signals
\item $F_s$-BL: space of bandlimited continuous-time signals, i.e., functions whose Fourier transform is zero outside of $[-F_s/2, F_s/2]$
\end{itemize}
\end{frame}
\begin{frame} \frametitle{Vector space: the ingredients}
The objects:
\begin{itemize}
\item the set of vectors $V$
\item a set of scalars (for us $\mathbb{C}$)
\end{itemize}
\vspace{2em}
The operations:
\begin{itemize}
\item resize vectors, i.e. multiply a vector by a scalar
\item combine vectors, i.e. sum them together
\item compare vectors, i.e. measure their difference
\end{itemize}
\end{frame}
\begin{frame} \frametitle{Linear combination in $\mathbb{R}^2$}
\begin{center}
$\mathbf{x} + \mathbf{y} = \myvector{x_0 + y_0 & x_1 + y_1}^T$
\vspace{2em}
\psset{unit=8mm}
\begin{pspicture}(-4,-1)(8,4)
\psgrid[subgriddiv=1,gridlabels=0pt,gridcolor=gray,griddots=10](0,0)(-4,-1)(8,4)
\psaxes[labelFontSize=\scriptstyle]{->}(0,0)(-4,-1)(8,4)
\psline[linecolor=darkred]{->}(5,1)\uput[-45](5,1){$\mathbf{x}$}
\psline[linecolor=darkred]{->}(2,2)\uput[45](2,2){$\mathbf{y}$}
\psline[linecolor=gray,linestyle=dashed](5,1)(7,3)
\psline[linecolor=gray,linestyle=dashed](2,2)(7,3)
\psline[linecolor=blue]{->}(7,3)\uput[45](7,3){$\mathbf{x+y}$}
\end{pspicture}
\end{center}
\end{frame}
\begin{frame} \frametitle{The inner (or dot) product}
\begin{center}
$\langle \cdot, \cdot \rangle \,:\, V \times V \rightarrow \mathbb{C}$
\end{center}
\begin{itemize}
\item measure of similarity between vectors
\item inner product is zero? vectors are \textit{orthogonal} (maximally different)
\item self inner product defines the \textit{norm}, that is, the size of a vector:
\[
\langle \mathbf{x}, \mathbf{x} \rangle = \|\mathbf{x}\|^2
\]
\end{itemize}
\end{frame}
\begin{frame} \frametitle{Inner product definitions}
\begin{align*}
\langle \mathbf{x} , \mathbf{y} \rangle &= \sum_{n = 0}^{N-1} x^*[n] y[n] & \mathbb{C}^N \\
&= \sum_{n = -\infty}^{\infty} x^*[n] y[n] & \ell_2(\mathbb{Z})\\
&= \int_{A}^{B} x^*(t)y(t)\, dt & L_2([A,B]) \\
&= \int_{-\infty}^{\infty} x^*(t)y(t)\, dt & L_2(\mathbb{Z})
\end{align*}
\centering
let's use geometric intuition to make sense of these definitions...
\end{frame}
\begin{frame} \frametitle{Inner product in $\mathbb{R}^2$}
\begin{center}
$\langle \mathbf{x}, \mathbf{y} \rangle = x_0 y_0 + x_1 y_1$
\vspace{2em}
\psset{unit=8mm}
\begin{pspicture}(-1,-1)(9,5)
\psgrid[subgriddiv=1,gridlabels=0pt,gridcolor=gray,griddots=10](0,0)(-1,-1)(9,5)
\psaxes[labelFontSize=\scriptstyle]{->}(0,0)(-1,-1)(9,5)
\psline{->}(6,2)\uput[-45](6,2){$\mathbf{x}$}
\psline{->}(3,4)\uput[-45](3,4){$\mathbf{y}$}
\end{pspicture}
\end{center}
\end{frame}
\begin{frame} \frametitle{Inner product in $\mathbb{R}^2$: the norm}
\begin{center}
$\langle \mathbf{x}, \mathbf{x} \rangle = x_0^2 + x_1^2 = \|\mathbf{x}\|^2$
\vspace{2em}
\psset{unit=8mm}
\begin{pspicture}(-1,-1)(9,5)
\psgrid[subgriddiv=1,gridlabels=0pt,gridcolor=gray,griddots=10](0,0)(-1,-1)(9,5)
\psaxes[labelFontSize=\scriptstyle]{->}(0,0)(-1,-1)(9,5)
\psline{->}(6,2)\uput[-45](6,2){$\mathbf{x}$}
\uput{3.2}[33]{0}(0,0){$\|\mathbf{x}\|$}
\end{pspicture}
\end{center}
\end{frame}
\begin{frame} \frametitle{Inner product in $\mathbb{R}^2$}
\begin{center}
$
\langle \mathbf{x}, \mathbf{y} \rangle = x_0 y_0 + x_1 y_1 = \|\mathbf{x}\| \, \|\mathbf{y}\| \, \cos\alpha
$
\vspace{2em}
\psset{unit=8mm}
\begin{pspicture}(-1,-1)(9,5)
\psgrid[subgriddiv=1,gridlabels=0pt,gridcolor=gray,griddots=10](0,0)(-1,-1)(9,5)
\psaxes[labelFontSize=\scriptstyle]{->}(0,0)(-1,-1)(9,5)
\psline{->}(6,2)\uput[-45](6,2){$\mathbf{x}$}
\psline{->}(3,4)\uput[-45](3,4){$\mathbf{y}$}
\psarc[linecolor=darkred]{->}{3}{18.435}{53.13}
\uput{3.2}[33]{0}(0,0){$\alpha$}
\end{pspicture}
\end{center}
\end{frame}
\begin{frame} \frametitle{Inner product in $\mathbb{R}^2$: orthogonality}
\begin{center}
$\langle \mathbf{x}, \mathbf{y} \rangle = x_0 y_0 + x_1 y_1= \|\mathbf{x}\|\,\|\mathbf{y}\|\,\cos\alpha$
\vspace{2em}
\psset{unit=8mm}
\begin{pspicture}(-1,0)(8,6)
\SpecialCoor%
\def\a{10 }\def\r{6 }
\def\b{100 }\def\s{5 }
\psline{->}(! \a cos \r mul \a sin \r mul)
\uput{\r}[\a](0,0){~$\mathbf{x}$}
\psline{->}(! \b cos \s mul \b sin \s mul)
\uput{\s}[\b](0,0){~$\mathbf{y}$}
\psarc[linecolor=darkred]{->}{3}{\a}{\b}
\uput{3.1}[55]{0}(0,0){$\alpha = \pi/2$}
\uput{5}[30]{0}(0,0){$\langle \mathbf{x}, \mathbf{y} \rangle = 0$}
\NormalCoor%
\end{pspicture}
\end{center}
\end{frame}
\begin{frame} \frametitle{Inner product in other spaces}
\centering
let's use the geometric intuition developed in Euclidean space\\ to understand inner product and norms in more abstract vector spaces
\end{frame}
\begin{frame} \frametitle{Inner product in $L_2[-1,1]$}
\begin{center}
$\mathbf{x}, \mathbf{y}$ from orthogonal subspaces: $\langle \mathbf{x}, \mathbf{y} \rangle = 0$
\vspace{1em}
\begin{figure}
\begin{dspPlot}[xticks=1,xout=true,yticks=2,sidegap=0]{-1, 1}{-1.25, 1.25}
\moocStyle
\dspText(.2,-.8){$\mathbf{x} = \sin(\pi t)$, antisymmetric}
\dspText(-.6,.8){$\mathbf{y} = 1-|t|$, symmetric}
\only<1>{
\dspFunc{x 180 mul sin}
\dspFunc[linecolor=orange]{x abs 1 exch sub }}
\only<2>{
\dspFunc[linecolor=lightgray]{x 180 mul sin}
\dspFunc[linecolor=lightgray]{x abs 1 exch sub }
\pscustom[fillstyle=solid,fillcolor=green!30,linestyle=none]{%
\dspFunc[linecolor=green]{x 180 mul sin x abs 1 exch sub mul dup 0 ge {} {pop 0} ifelse}}
\pscustom[fillstyle=solid,fillcolor=red!30,linestyle=none]{%
\dspFunc[linecolor=red]{x 180 mul sin x abs 1 exch sub mul dup 0 ge {pop 0} {} ifelse}}}
\end{dspPlot}
\end{figure}
\end{center}
\end{frame}
\begin{frame} \frametitle{Inner product in $L_2[-1,1]$}
\begin{center}
sinusoids with frequencies integer multiples of a fundamental
\vspace{1em}
\begin{figure}
\begin{dspPlot}[xticks=1,xout=true,yticks=2,sidegap=0]{-1, 1}{-1.2, 1.2}
\moocStyle
\dspText(1.3,-.5){\color{darkred} $\mathbf{x} = \sin(4\pi t)$}
\dspText(1.3,.5){\color{orange} $\mathbf{y} = \sin(5\pi t)$}
\only<1>{\dspFunc{x 180 mul 4 mul sin}}
\only<1>{\dspFunc[linecolor=orange]{x 180 mul 5 mul sin}}
\only<2>{
\dspFunc[linecolor=lightgray]{x 180 mul 4 mul sin}
\dspFunc[linecolor=lightgray]{x 180 mul 5 mul sin}
\pscustom[fillstyle=solid,fillcolor=green!30,linestyle=none]{%
\dspFunc[linecolor=green]{x 180 mul 4 mul sin x 180 mul 5 mul sin mul dup 0 ge {} {pop 0} ifelse}}
\pscustom[fillstyle=solid,fillcolor=red!30,linestyle=none]{%
\dspFunc[linecolor=red]{x 180 mul 4 mul sin x 180 mul 5 mul sin mul dup 0 ge {pop 0} {} ifelse}}}
\end{dspPlot}
\end{figure}
\end{center}
\end{frame}
\begin{frame} \frametitle{Norm as a measure of energy}
\begin{align*}
\langle \mathbf{x}, \mathbf{x} \rangle &= \sum |x[n]|^2 \\
&= \int |x(t)|^2 \, dt
\end{align*}
\centering
think of $x$ as voltage across a $1~\Omega$ resistor
\end{frame}
\begin{frame} \frametitle{Norm and distance in $\mathbb{R}^2$}
\begin{center}
More geometric intuition:
\vspace{2em}
$\|\mathbf{x} - \mathbf{y}\| = \sqrt{(x_0 - y_0)^2 + (x_1 - y_1)^2}$
\psset{unit=8mm}
\begin{pspicture}(-1,0)(8,6)
\psline{->}(6,1)\uput[-45](3,0.5){$\|\mathbf{x}\|$}
\psline{->}(4,3)\uput[135](2,1.5){$\|\mathbf{y}\|$}
\psline[linecolor=blue]{->}(4,3)(6,1)\uput[45](5,2){$\|\mathbf{x-y}\|$}
\psline[linewidth=0.5pt,linestyle=dashed](0, 1)(7,1)
\psline[linewidth=0.5pt,linestyle=dashed](7,3)(0,3)
\psline[linewidth=0.5pt,linestyle=dashed](6,0)(6,2)
\psline[linewidth=0.5pt,linestyle=dashed](4,0)(4,4)
\end{pspicture}
\end{center}
\end{frame}
\begin{frame} \frametitle{Distance for signals: the Mean Square Error}
\[
\|\mathbf{x} - \mathbf{y}\|^2 = \sum_n |x[n] - y[n]|^2 \quad \leftarrow \text{energy of the difference}
\]
\end{frame}
\begin{frame} \frametitle{The concept of basis}
Given:
\begin{itemize}
\item a vector space $H$
\item a set of $K$ vectors from $H$: $W = \{ \mathbf{w}^{(k)} \}_{k =0,1,\ldots, K-1}$
\end{itemize}
\vspace{1em}
$W$ is a basis for $H$ if:
\begin{enumerate}
\item we can write for {\em all} $\mathbf{x} \in H$:
\[
\mathbf{x} = \sum_{k=0}^{K-1} \alpha_k \mathbf{w}^{(k)}, \quad \alpha_k \in \mathbb{C}
\]
\item the coefficients $\alpha_k$ are unique
\end{enumerate}
\end{frame}
\begin{frame} \frametitle{The canonical $\mathbb{R}^2$ basis}
\psset{unit=15mm}
\begin{columns}
\begin{column}{.4\paperwidth}
\[ \mathbf{x} = \myvector{2 \\ 1} \]
\vspace{1em}
\[ \mathbf{x} = 2\mathbf{e}^{(0)} + \mathbf{e}^{(1)} \]
\end{column}
\begin{column}{.6\paperwidth}
\centering
\begin{pspicture}(-1,-1)(3,3)
{\color{lightgray}
\footnotesize
\psaxes[linewidth=0.5pt,linecolor=lightgray](0,0)(-1.5,-1.5)(2.5,2,5)}
\psline{->}(2,1)
\uput[45](2,1){$\mathbf{x}$}
\psline[linecolor=green]{->}(1,0)\rput[tr]{0}(1,-0.1){$\mathbf{e}^{(0)}$}
\psline[linecolor=green]{->}(0,1)\rput[tr]{0}(.5,1){$\mathbf{e}^{(1)}$}
\end{pspicture}
\end{column}
\end{columns}
\end{frame}
\begin{frame} \frametitle{Another basis for $\mathbb{R}^2$}
\psset{unit=15mm}
\begin{columns}
\begin{column}{.4\paperwidth}
\[ \mathbf{x} = \myvector{2 \\ 1} \]
\vspace{1em}
\[ \mathbf{x} = 2\mathbf{e}^{(0)} + \mathbf{e}^{(1)} \]
\vspace{1em}
\[ \mathbf{x} = \sqrt{5}\mathbf{v}^{(0)} \]
\centering
50\% data compression!
\end{column}
\begin{column}{.6\paperwidth}
\centering
\begin{pspicture}(-1,-1)(3,3)
{\color{lightgray}
\footnotesize
\psaxes[linewidth=0.5pt,linecolor=lightgray](0,0)(-1.5,-1.5)(2.5,2,5)}
\psline{->}(2,1)
\uput[45](2,1){$\mathbf{x}$}
\psline[linecolor=green]{->}(1,0)\rput[tr]{0}(1,-0.1){$\mathbf{e}^{(0)}$}
\psline[linecolor=green]{->}(0,1)\rput[tr]{0}(.5,1){$\mathbf{e}^{(1)}$}
\uput*{0.65}[15]{0}(0,0){$\theta$}%
\psarc[linewidth=0.2pt,linecolor=gray]{->}{0.6}{0}{30}
\psline[linecolor=blue]{->}(! 26.565 cos 26.565 sin)\rput[tl]{0}(! 30 cos 30 sin){$\mathbf{v}^{(0)}$}
\psline[linecolor=blue]{->}(! 116.565 cos 116.565 sin)\rput[br]{0}(!120 cos 120 sin){$\mathbf{v}^{(1)}$}
\end{pspicture}
\end{column}
\end{columns}
\end{frame}
\begin{frame} \frametitle{Special bases}
\centering
Orthogonal basis:
\[
\langle \mathbf{w}^{(k)}, \mathbf{w}^{(n)} \rangle = 0 \mbox{ for } k \neq n
\]
\vspace{1em}
Orthonormal basis: orthogonal plus basis vectors have unit norm
\[
\langle \mathbf{w}^{(k)}, \mathbf{w}^{(k)} \rangle = 1
\]
\end{frame}
\begin{frame} \frametitle{Basis expansion}
\centering
\[
\mathbf{x} = \sum_{k=0}^{K-1} \alpha_k \mathbf{w}^{(k)}
\]
how do we find the $\alpha$'s ?
\vspace{2em}
Orthonormal bases are the best:
\[
\alpha_k = \langle \mathbf{w}^{(k)}, \mathbf{x} \rangle
\]
since
\[
\langle \mathbf{w}^{(k)}, \mathbf{x} \rangle = \langle \mathbf{w}^{(k)}, \sum_{n=0}^{K-1} \alpha_n \mathbf{w}^{(n)} \rangle = \sum_{n=0}^{K-1} \alpha_n \langle \mathbf{w}^{(k)}, \mathbf{w}^{(n)} \rangle = \alpha_k
\]
\end{frame}
\begin{frame} \frametitle{Change of basis (for orthonormal bases)}
\centering
\[
\mathbf{x} = \sum_{k=0}^{K-1} \alpha_k \mathbf{w}^{(k)} = \sum_{k=0}^{K-1} \beta_k \mathbf{v}^{(k)}
\]
if $\{ \mathbf{v}^{(k)} \}$ is orthonormal:
\begin{align*}
\beta_h &= \langle \mathbf{v}^{(h)}, \mathbf{x} \rangle \\
&= \langle \mathbf{v}^{(h)}, \sum_{k=0}^{K-1} \alpha_k \mathbf{w}^{(k)} \rangle \\
&= \sum_{k=0}^{K-1} \alpha_k \langle \mathbf{v}^{(h)}, \mathbf{w}^{(k)} \rangle
\end{align*}
\end{frame}
\begin{frame} \frametitle{Change of basis, orthonormal case}
\centering
\begin{align*}
\beta_h &= \sum_{k=0}^{K-1} \alpha_k \langle \mathbf{v}^{(h)}, \mathbf{w}^{(k)} \rangle = \sum_{k=0}^{K-1} \alpha_k \, c_{hk} \\ \\
\begin{bmatrix}
\beta_0 \\ \vdots \\ \beta_{K-1}
\end{bmatrix}
&= \begin{bmatrix}
c_{00} & c_{01} & \ldots & c_{0(K-1)} \\
& & \vdots & \\
c_{(K-1)0} & c_{(K-1)1} & \ldots & c_{(K-1)(K-1)} \\
\end{bmatrix}
\begin{bmatrix}
\alpha_0 \\ \vdots \\ \alpha_{K-1}
\end{bmatrix}
\end{align*}
\vspace{1em}
\centering
what you see here is the essence of the Fourier transform!
\end{frame}
\begin{frame} \frametitle{Parseval's Theorem}
\centering
If $\{ \mathbf{w}^{(k)} \}$ is orthonormal and $\displaystyle
\mathbf{x} = \sum_{k=0}^{K-1} \alpha_k \mathbf{w}^{(k)}$
\vspace{2em}
\[
\| \mathbf{x} \|^2 = \langle \mathbf{x}, \mathbf{x} \rangle = \sum_{k=0}^{K-1} |\alpha_k|^2
\]
energy is conserved across a change of basis
\end{frame}
\begin{frame} \frametitle{Least-squares approximation}
\begin{columns}
\begin{column}{.4\paperwidth}
Problem:
\begin{itemize}
\item vector $\mathbf{x} \in V$
\item subspace $S \subseteq V$
\vspace{1em}
\item approximate $\mathbf{x}$ with $\hat{\mathbf{x}} \in S$
\end{itemize}
\end{column}
\begin{column}{.5\paperwidth}
\centering
\psset{unit=3cm}
\begin{pspicture}(-1.5,-.8)(1.5,1.8)
\psset{Alpha=40,linewidth=1pt}
\pstThreeDCoor[linecolor=black,linewidth=1pt,%
xMax=1.5,yMax=1.5,zMax=1.5,%
xMin=-.2,yMin=-.2,zMin=-.2,%
nameX=$\mathbf{e}^{(0)}$,
nameY=$\mathbf{e}^{(1)}$,
nameZ=$\mathbf{e}^{(2)}$]
\pstThreeDLine[linewidth=1.8pt]{->}(0,0,0)(1,.5,2)\pstThreeDPut(1,.5,2.15){$\mathbf{x}$}
\pstThreeDLine[linecolor=darkred]{->}(0,0,0)(1.5,0,0)
\pstThreeDLine[linecolor=darkred]{->}(0,0,0)(0,1.5,0) \pstThreeDLine[linestyle=dashed,linewidth=0.5pt](0,.5,0)(1,.5,0)
\pstThreeDLine[linestyle=dashed,linewidth=0.5pt](1,.5,0)(1,0,0)
\pstThreeDLine[linestyle=dashed,linewidth=0.5pt](1,.5,0)(1,.5,2)
\pstThreeDLine[linecolor=gray,linewidth=1.8pt]{->}(0,0,0)(1,.5,0)
\pstThreeDPut(1.15,.65,0){$\hat{\mathbf{x}}$}
\end{pspicture}
\end{column}
\end{columns}
\end{frame}
\begin{frame} \frametitle{Orthogonal Projection}
\begin{itemize}
\item $\{ \mathbf{s}^{(k)} \}_{k =0,1,\ldots, K-1}$ orthonormal basis for $S$
\item orthogonal projection: $\hat{\mathbf{x}} = \sum_{k=0}^{K-1} \langle \mathbf{s}^{(k)}, \mathbf{x} \rangle \, \mathbf{s}^{(k)} $
\end{itemize}
\vspace{2em}
Orthogonal projection is the ``best'' approximation over $S$:
\begin{itemize}
\item the approximation error has minimum norm:
\[
\arg\min_{\mathbf{y} \in S} \| \mathbf{x} - \mathbf{y} \| = \hat{\mathbf{x}}
\]
\item the error is orthogonal to approximation (no residual information):
\[
\langle \mathbf{x} - \hat{\mathbf{x}}, \, \hat{\mathbf{x}} \rangle = 0
\]
\end{itemize}
\end{frame}
\begin{frame} \frametitle{Recap of signals and vector space}
\begin{itemize}
\item $\mathbb{C}^{N}$: space of finite-length (or periodic) discrete-time signals; equivalent to a Python array
\item $\ell_2(\mathbb{Z})$: space of infinite-length, finite-energy discrete-time signals; good for proofs
\item $L_2([-\pi, \pi])$: space of $2\pi$-periodic square-integrable complex functions of a real variable; this is where discrete-time Fourier transforms live
\item $L_2(\mathbb{R})$: space of finite-energy continuous-time signals; idealized analog signals
\item $F_s$-BL: space of bandlimited continuous-time signals, i.e., functions whose Fourier transform is zero outside of $[-F_s/2, F_s/2]$
\end{itemize}
\end{frame}
\end{document}
c4science · Help