%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% ------------------------------------------------------------------------
% Large Sample Theory for MLEs *******************************************
% ------------------------------------------------------------------------
% Submitted:      Dr. Eric Suess, Statistics 6502, Spring 2006
% ------------------------------------------------------------------------
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentclass[12]{report}

\usepackage{latexsym}
\usepackage[centertags]{amsmath}
\usepackage{amsfonts}
\usepackage{newlfont}
%\usepackage{graphicx,fullpage,subfigure,pstricks}

\setlength{\oddsidemargin}{-0.4cm}
\setlength{\evensidemargin}{0cm}
\setlength{\topmargin}{-0.75cm}
\setlength{\textheight}{8.9in}
\setlength{\textwidth}{6.5in}
\renewcommand{\baselinestretch}{1}
\setlength{\parskip}{3mm}
\setlength{\parindent}{0mm} % indented paragraphs just don't

%\pagestyle{plain}

\begin{document}
\pagestyle{myheadings} \markright{Statistics 6502, Spring 2007}

{\large
\begin{center}
\textbf{Large Sample Theory for MLEs}
\end{center}}

\textbf{Introduction}

Consider the following:

\begin{itemize}
\item i.i.d. sample $X_1,X_2,...,X_n$
\item one-dimensional paramter $\theta$ with $l(\theta) = \sum_{i=1}^n log(f(x_i|\theta)$
\item the true value of $\theta$ is $\theta_0$
\end{itemize}

{\textbf{\underline{Thm A:}}} Under appropriate smoothness conditions on $f$, the MLE of $\theta$ from an i.i.d. sample is consistent, i.e.,
\begin{equation}
\hat{\theta} \xrightarrow{p} \theta_0, \ \ n \rightarrow \infty.
\end{equation}

{\textbf{\underline{Proof:}}}  (The idea)

First, think of the transformed variable $log f(X|\theta)$ and consider $ log f(X_1|\theta), log f(X_2|\theta), ..., log f(X_n|\theta)$ as a random sample.  So its mean is

\begin{equation}
\frac{1}{n} \sum_{i=1}^n log f(X_i|\theta) \xrightarrow{p} E_{\theta_0}[log f(X_i|\theta)] \ \ n \rightarrow \infty.
\end{equation}

The subscript on $\theta$, $\theta_0$, is to identify the true unknown value of $\theta$.

With some imagination (or doing Calculus) we can guess the maximum of the righthand side should be at $\theta_0$ since the convergence says that the lefthand side and the righthand side will be close for large $n$.

{\textbf{\underline{Lemma A:}}} Define $I(\theta)$, Fisher's Information, by

\begin{equation}
I(\theta) = E\left[\frac{d}{d \theta} log f(X|\theta)\right]^2.
\end{equation}

Under the appropriate smoothness conditions on $f$, $I(\theta)$ may be represented as

\begin{equation}
I(\theta) = -E\left[\frac{d^2}{d \theta^2} log f(X|\theta)\right].
\end{equation}

{\textbf{\underline{Proof:}}}

First recall that for any density funtion

\begin{equation}
\int f(x|\theta) dx = 1
\end{equation}

and so

\begin{equation}
\frac{d}{d \theta} \int f(x|\theta) dx = 0.
\end{equation}

Note that since

\begin{equation}
\frac{d}{dx} log(g(x)) = \frac{\frac{d}{dx} g(x)}{g(x)},
\end{equation}

then

\begin{equation}
\frac{d}{d \theta} f(x|\theta) = \left[\frac{d}{d \theta} log f(x|\theta) \right] f(x|\theta).
\end{equation}

So

\begin{eqnarray*}
0 &=& \frac{d}{d \theta} \int f(x|\theta) dx \\
&=& \int \frac{d}{d \theta} f(x|\theta) dx \\
&=& \int \left[\frac{d}{d \theta} log f(x|\theta) \right] f(x|\theta) dx
\end{eqnarray*}

The moving of the derivative inside the integral takes some math.  Now take the second derivative.

\begin{eqnarray*}
0 &=& \frac{d}{d \theta} \int \left[\frac{d}{d \theta} log f(x|\theta) \right] f(x|\theta) dx \\
&=& \int \left[\frac{d^2}{d\theta^2} log f(x|\theta) \right] f(x|\theta) dx +
    \int \left[\frac{d}{d\theta} log f(x|\theta) \right] \frac{d}{d\theta} f(x|\theta) dx \\
&=& \int \left[\frac{d^2}{d\theta^2} log f(x|\theta) \right] f(x|\theta) dx +
    \int \left[\frac{d}{d\theta} log f(x|\theta) \right]^2 f(x|\theta) dx \\
&=& E\left[\frac{d^2}{d\theta^2} log f(X|\theta) \right] + E\left[\frac{d}{d\theta} log f(X|\theta) \right]^2.
\end{eqnarray*}

\textbf{Remarks about Fisher's Information}

For the one-parameter model $f(x|\theta)$ the Fisher Information $I(\theta)$ is the fundamental measure of the ``information'' an observation $X$ carries about $\theta$.

\begin{equation}
E\left[\frac{d^2}{d\theta^2} log f(X|\theta) \right] = -E\left[\frac{d}{d\theta} log f(X|\theta) \right]^2.
\end{equation}

\begin{enumerate}
\item
$I_X(\theta)$ is best thought of in relative terms, i.e., $X$ is more informative about $\theta$ than $Y$ if
$I_X(\theta) > I_Y(\theta)$.
\item
The more information that $X$ contains about $\theta$, the larger $I_X(\theta)$ should be.  If $X = \theta$ with probability one, then we would like to have $I_X(\theta) = \infty$.
\item
If $X_1, X_2, ..., X_n$ i.i.d. $F_{\theta}$, then
\begin{equation}
I_{\mathbf X}(\theta) = n I_X(\theta)
\end{equation}
where ${\mathbf X} = (X_1, X_2, ..., X_n)$.
\end{enumerate}

The information of $n$ $X$'s is $n$ times the information from one $X$ and as $n \rightarrow \infty$, the information goes to $\infty$, and is eventually perfect.

{\textbf{\underline{Proof:}}}

\begin{eqnarray*}
I_{\mathbf X}(\theta) &=& -E\left[\frac{d^2}{d\theta^2} log f(X_1,X_2, ..., X_n|\theta)\right] \\
&=& -E\left[\frac{d^2}{d\theta^2} log \prod_{i=1}^n f(X_i|\theta)\right] \\
&=& -E\left[\frac{d^2}{d\theta^2} \sum_{i=1}^n log f(X_i|\theta)\right] \\
&=& \sum_{i=1}^n \left[ -E\left[\frac{d^2}{d\theta^2} log f(X_i|\theta)\right] \right] \\
&=& \sum_{i=1}^n I_X(\theta) \\
&=& n I_X(\theta)
\end{eqnarray*}

The large sample distribution of an MLE is approximately normal with mean $\theta_0$ and variance $\frac{1}{nI(\theta_0)}$.  This implies that the MLE is asymptotically unbiased.  It also implies that the variance of the limiting normal distribution has the asymptotic variance of the MLE.

{\textbf{\underline{Thm B:}}} Under the smoothness conditions of $f$, the probability distribution of

\begin{equation}
\sqrt{nI(\theta_0)}(\hat{\theta} - \theta_0)
\end{equation}

tends to a standard normal distribution.

{\textbf{\underline{Proof:}}}

Use the W.L.L.N., the C.L.T. and Lemma A.

So we have

\begin{equation}
\hat{\theta} \sim N\left(\theta_0,\frac{1}{nI(\theta_0)}\right)
\end{equation}

and approximately

\begin{equation}
\hat{\theta} \sim N\left(\theta_0,\frac{1}{nI(\hat{\theta})}\right)
%\hat{\theta} \sim N\left(\theta_0,\frac{1}{nI(\hat{\theta})}}\right)
\end{equation}

from which we can produce an approximate $100(1-\alpha)$ confidence interval for $\theta$


\begin{equation}
\hat{\theta} \pm z_{\alpha/2} \frac{1}{\sqrt{nI(\hat{\theta})}}
\end{equation}

\textbf{Notes about the smoothness conditions}

The true parameter value, $\theta_0$, is required to be an interior point of the set of all parameter values.

It is also required that the support of the density $f(x|\theta)$ does not depend of $\theta$.

\textbf{Final notes}

The asymptotic variance of the MLE $\hat{\theta}$ is

\begin{equation}
\frac{1}{n I(\theta_0)} = -\frac{1}{E\left[\frac{d^2}{d\theta^2} l(\theta)\right]}.
\end{equation}

\end{document}
% ------------------------------------------------------------------------
