# HG changeset patch
# User Oleksandr Gavenko <gavenkoa@gmail.com>
# Date 1461244838 -10800
# Node ID c48b0353e055e005155146ada871b128162c6f4d
# Parent  f606de7b91b0d146f808b9cd0840b35dd489153d
Add "Law of total expectation", "Law of total variance", proofs for covariance.

diff -r f606de7b91b0 -r c48b0353e055 probability-continuous.rst
--- a/probability-continuous.rst	Thu Apr 21 16:19:13 2016 +0300
+++ b/probability-continuous.rst	Thu Apr 21 16:20:38 2016 +0300
@@ -16,7 +16,7 @@
 
 .. math::
 
-   PDF(a ≤ X ≤ b) = P(a ≤ X ≤ b) = ∫_{a, b}\ f_X(x) \ dx
+   CDF(a ≤ X ≤ b) = P(a ≤ X ≤ b) = ∫_{a, b}\ f_X(x) \ dx
 
    f_X(x) ≥ 0
 
@@ -273,9 +273,9 @@
 
 .. math::
 
-   f_Y(y) = 1/a·f_X(y/a) = 1/a·1/sqrt(2·π)/σ·e^{-λ·((y-b)/a - μ)²/σ²/2}
+   f_Y(y) = 1/a·f_X((y-b)/a) = 1/a·1/sqrt(2·π)/σ·e^{-λ·((y-b)/a - μ)²/σ²/2}
 
-   = 1/sqrt(2·π)/(a·σ)·e^{-λ·(y - (a·μ+b))²/(a·σ)²/2} = ~ norm(a·μ+b, a·σ)
+   = 1/sqrt(2·π)/(a·σ)·e^{-λ·(y - (a·μ+b))²/(a·σ)²/2} = ~ norm(a·μ+b, (a·σ)²)
 
 Monotonic function of distribution
 ==================================
@@ -320,6 +320,8 @@
 
 .. math:: f_Z(z) = ∫_x\ f_{X,Z}(x,z)\ dx = ∫_x\ f_X(x)·f_Y(z-x)\ dx
 
+* https://en.wikipedia.org/wiki/List_of_convolutions_of_probability_distributions
+
 Covariance
 ==========
 
@@ -341,11 +343,45 @@
 
 Covariance of two independent r.v. is zero.
 
-Proof:
+Proofs:
+
+.. math::
+
+   cov(X, Y) = E[(X - E[X])·(Y - E[Y])] = E[ X·Y - X·E[Y] - E[X]·Y + E[X]·E[Y] ]
+
+   = E[X·Y] - E[X·E[Y]] - E[E[X]·Y] + E[E[X]·E[Y]]
+
+   = E[X·Y] - E[X]·E[Y] - E[X]·E[Y] + E[X]·E[Y] = E[X·Y] - E[X]·E[Y]
+
+.. math::
+
+   cov(a·X + b, Y) = E[(a·X + b - E[a·X + b])·(Y - E[Y])]
+
+   = E[(a·X + b - (a·E[X] + b))·(Y - E[Y])] = E[(a·X + a·E[X])·(Y - E[Y])]
+
+   = a·E[(X + E[X])·(Y - E[Y])] = a·cov(X, Y)
 
 .. math::
 
-   cov(X, Y) = E[(X - E[X])·(Y - E[Y])] = E[(X - E[X])]·E[(Y - E[Y])] = 0
+   cov(X, Y + Z) = E[(X - E[X])·(Y + Z - E[Y + Z])] = E[(X - E[X])·(Y - E[Y] + Z - E[Z])]
+
+   = E[(X - E[X])·(Y - E[Y]) + (X - E[X])·(Z - E[Z])]
+
+   = E[(X - E[X])·(Y - E[Y])] + E[(X - E[X])·(Z - E[Z])] = cov(X, Y) + cov(X, Z)
+
+.. math::
+
+   var(X) + var(Y) + 2·cov(X, Y) = E[X²] - (E[X])² + E[Y²] - (E[Y])² + 2·E[X·Y] - 2·E[X]·E[Y]
+
+   = E[X² - X·E[X] + Y² - Y·E[Y] + 2·X·Y - X·E[Y] - Y·E[X]]
+
+   = E[(X+Y)² - (X·E[X] + Y·E[Y] + X·E[Y] + Y·E[X])]
+
+   = E[(X+Y)²] - E[(X+Y)·(E[X] + E[Y])] = E[(X+Y)²] - E[X+Y]·E[X+Y] = var(X+Y)
+
+For independent r.v. :math:`X` and :math:`Y`:
+
+.. math:: cov(X, Y) = E[(X - E[X])·(Y - E[Y])] = E[(X - E[X])]·E[(Y - E[Y])] = 0
 
 Correlation coefficient
 =======================
@@ -367,3 +403,69 @@
 
 .. math:: ρ(a·X + b, Y) = sign(a)·ρ(X, Y)
 
+Conditioned expectation
+=======================
+
+.. math:: E[X|Y] = ∫_X\ x·f_{X|Y}(x|Y)\ dx
+
+Law of total expectation
+========================
+
+.. math:: E[X] = E[E[X|Y]]
+
+Proof::
+
+.. math::
+
+   E[E[X|Y]] = ∫_Y\ f_Y(y)·∫_X\ x·f_{X|Y}(x|y)\ dx·dy
+
+   = ∫_Y\ ∫_X\ x·f_Y(y)·f_{X|Y}(x|y)\ dx·dy = ∫_Y\ ∫_X\ x·f_{X,Y}(x,y)\ dx·dy
+
+   = ∫_X\ x·∫_Y\ f_{X,Y}(x,y)\ dy·dx = ∫_X\ x·f_X(x)\ dx = E[X]
+
+* https://en.wikipedia.org/wiki/Law_of_total_expectation
+
+Iterated expectations with nested conditioning sets
+===================================================
+
+.. math:: E[X|A] = E[E[X|B]|A]
+
+Conditional variance
+====================
+
+.. math:: var(X|Y=y) = E[(X - E[X|Y=y])² |Y=y]
+
+* https://en.wikipedia.org/wiki/Conditional_variance
+
+Law of total variance
+=====================
+
+.. math:: var(X) = E[var(X|Y)] + var(E[X|Y]) = E_Y[var_X(X|Y)] + var_X(E_Y[X|Y])
+
+Proof:
+
+.. math::
+
+   var(X) = E[X²] - (E[X])² = E[E[X²|Y]] - (E[E[X|Y]])²
+
+   = E[var(X|Y) + (E[X|Y])²] - (E[E[X|Y]])²
+
+   = E[var(X|Y)] + E[(E[X|Y])²} - (E[E[X|Y]])² = E[var(X|Y)] + var(E[X|Y])
+
+* https://en.wikipedia.org/wiki/Law_of_total_variance
+
+Law of total covariance
+=======================
+
+* https://en.wikipedia.org/wiki/Law_of_total_covariance
+
+Sum of normally distributed random variables
+============================================
+
+For :math:`X ~ norm(μ_X, σ_X²)` and :math:`Y ~ norm(μ_Y, σ_Y²)` random variable
+:math:`X+Y` is also has normal distribution with parameters:
+
+.. math:: norm(μ_X + μ_Y, σ_X² + σ_Y²)
+
+https://en.wikipedia.org/wiki/Sum_of_normally_distributed_random_variables
+