diff --git a/is/UB3/ExerciseSheet3.ipynb b/is/UB3/ExerciseSheet3.ipynb index dd7e77f..9e8a3ca 100644 --- a/is/UB3/ExerciseSheet3.ipynb +++ b/is/UB3/ExerciseSheet3.ipynb @@ -19,12 +19,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "#Intelligent Systems Sheet 3\n", + "Maximus Mutschler & Jan-Peter Hohloch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "# Gaussian Algebra (25 Points)\n", "\n", "Prove that the product of two univariate (scalar) Gaussian distributions is a Gaussian again, i.e. show, by explicitly performing the required arithmetic transformations, that\n", "\n", "\\begin{equation}\n", - " \\normal(x;\\mu,\\sigma^2)\\normal(x;m,s^2) = \\normal[x; (\\frac 1{\\sigma^2}+\\frac 1{s^2})^{-1}(\\frac \\mu{\\sigma^2}+\\frac m{s^2}),(\\frac 1{\\sigma^2}+\\frac 1{s^2})^{-1}]\\normal[m,\\mu,\\sigma^2+s^2].\n", + " \\normal(x;\\mu,\\sigma^2)\\normal(x;m,s^2) = \\normal\\left[x; \\left(\\frac 1{\\sigma^2}+\\frac 1{s^2}\\right)^{-1}\\left(\\frac \\mu{\\sigma^2}+\\frac m{s^2}\\right),\\left(\\frac 1{\\sigma^2}+\\frac 1{s^2}\\right)^{-1}\\right]\\normal\\left[\\mu;m,\\sigma^2+s^2\\right].\n", "\\end{equation}" ] }, @@ -32,6 +40,30 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "## Proof\n", + "\\begin{align*}\n", + " f(x) &= \\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\cdot e^{\\frac{-(x-\\mu)^2}{2\\sigma^2}}\\cdot \\frac{1}{\\sqrt{2\\pi s^2}}\\cdot e^{\\frac{-(x-m)^2}{2s^2}}\\\\\n", + " &=\\frac{1}{2\\pi s\\sigma}\\cdot e^{-\\alpha}\\\\\n", + " \\text{where }\\alpha &= \\frac{s^2(x-\\mu)^2+\\sigma^2(x-m)^2}{2\\sigma^2 s^2}\\\\\n", + " &=\\frac{(s^2+\\sigma^2)x^2-2(s^2\\mu+\\sigma^2 m)x+s^2\\mu^2+\\sigma^2m^2}{2\\sigma^2s^2}\\\\\n", + " &=\\frac{x^2-2\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2}x+\\frac{s^2\\mu^2+\\sigma^2m^2}{s^2+\\sigma^2}}{\\frac{2\\sigma^2s^2}{s^2+\\sigma^2}}\\\\\n", + " &=\\frac{\\left(x-\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2}\\right)^2-\\left(\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2}\\right)^2+\\frac{s^2\\mu^2+\\sigma^2m^2}{s^2+\\sigma^2}}{\\frac{2\\sigma^2s^2}{s^2+\\sigma^2}}\\\\\n", + " &=\\frac{\\left(x-\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2}\\right)^2}{\\frac{2\\sigma^2s^2}{s^2+\\sigma^2}}+\\underbrace{\\frac{\\frac{s^2\\mu^2+\\sigma^2m^2}{s^2+\\sigma^2}-\\left(\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2}\\right)^2}{\\frac{2\\sigma^2s^2}{s^2+\\sigma^2}}}_\\beta\\\\\n", + " \\beta&= \\frac{\\frac{s^4\\mu^2+s^2\\sigma^2\\mu^2+s^2\\sigma^2m^2+\\sigma^4m^2-\\left(s^4\\mu^2+2s^2\\sigma^2\\mu m+\\sigma^4m^2\\right)}{\\left(s^2+\\sigma^2\\right)^2}}{\\frac{2\\sigma^2s^2}{s^2+\\sigma^2}}\\\\\n", + " &=\\frac{s^2\\sigma^2\\left(\\mu^2-2\\mu m+m^2\\right)}{2\\sigma^2s^2\\left(s^2+\\sigma^2\\right)}\\\\\n", + " &=\\frac{(\\mu-m)^2}{2(s^2+\\sigma^2)}\\\\\n", + " \\Rightarrow f(x)&=\\frac{1}{2\\pi s\\sigma}\\cdot e^{-(\\alpha-\\beta)-\\beta}\\\\\n", + " &=\\frac{1}{2\\pi s\\sigma}\\cdot e^{-\\frac{\\left(x-\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2}\\right)^2}{\\frac{2\\sigma^2s^2}{s^2+\\sigma^2}}}\\cdot e^{-\\frac{(\\mu-m)^2}{2(s^2+\\sigma^2)}}\\\\\n", + " &=\\frac{\\sqrt{2\\pi(s^2+\\sigma^2)}\\sqrt{2\\pi\\frac{\\sigma^2s^2}{s^2+\\sigma^2}}}{2\\pi s\\sigma}\\normal\\left[x;\\frac{s^2\\mu+\\sigma^2 m}{s^2+\\sigma^2},\\frac{\\sigma^2s^2}{s^2+\\sigma^2}\\right]\\normal\\left[\\mu;m,\\sigma^2+s^2\\right]\\\\\n", + " &=\\frac{\\sqrt{2\\pi\\cdot 2\\pi\\sigma^2s^2}}{2\\pi s\\sigma}\\normal\\left[x; \\left(\\frac 1{\\sigma^2}+\\frac 1{s^2}\\right)^{-1}\\left(\\frac \\mu{\\sigma^2}+\\frac m{s^2}\\right),\\left(\\frac 1{\\sigma^2}+\\frac 1{s^2}\\right)^{-1}\\right]\\normal\\left[\\mu;m,\\sigma^2+s^2\\right]\\\\\n", + " &=\\normal\\left[x; \\left(\\frac 1{\\sigma^2}+\\frac 1{s^2}\\right)^{-1}\\left(\\frac \\mu{\\sigma^2}+\\frac m{s^2}\\right),\\left(\\frac 1{\\sigma^2}+\\frac 1{s^2}\\right)^{-1}\\right]\\normal\\left[\\mu;m,\\sigma^2+s^2\\right]\\hspace{2cm} q.e.d.\n", + "\\end{align*}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "# Maximum Likelihood Estimator of Simple Linear Regression (25 Points)\n", "\n", "Derive the formula $\\mathbf{w}_{MLE} = (X^TX)^{-1}X^T\\mathbf{y}$ from the lecture, by calculating the derivative of $p(\\mathbf{y}\\,|X,\\mathbf{w}) = \\normal(\\mathbf{y}\\,|X\\mathbf{w}, \\sigma^2I)$ with respect to $\\mathbf{w}$, setting it to zero and solving it for $\\mathbf{w}$.\n", @@ -307,21 +339,21 @@ }, "celltoolbar": "Edit Metadata", "kernelspec": { - "display_name": "Python 2", + "display_name": "Python 3", "language": "python", - "name": "python2" + "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.8" + "pygments_lexer": "ipython3", + "version": "3.4.3" } }, "nbformat": 4,