\documentstyle[bj,txfonts]{article}

\frenchspacing
\def\vtrim{\vspace*{-0.8em}}

\begin{document}

\begin{center}
\Large
36-617: Applied Linear Models\\
Fall 2022\\
HW02 -- Due Mon Sept 12, 11:59pm
\end{center}

\begin{itemize}\parsep=0pt\itemsep=0pt\parskip=0pt

\item Please turn the homework in online in our course webspace at canvas.cmu.edu.
\begin{itemize}
\item There is a link to Gradescope in the description of this assignment on Canvas. 
\item You should submit a single pdf to Gradescope. If you need help with this, please see \\ https://www.cmu.edu/teaching/gradescope/index.html.  Also, allow yourself some extra time to create the pdf \& upload it in Gradescope.
\item Don't forget to identify which pages each (part of each) problem appears on in your solitions.  Gradescope allows the TA to grade all the problem 1's together, then all the problem 2's, and so forth.  This leads to more consistent grading and better comments for you.
\item Remember to list who you worked with, on this and every assignment.
\end{itemize}

\item Reading:
\begin{itemize}

\item For next week: Sheather Ch 6 (supplemental: ISLR 3.3.3; G\&H Ch 4)

\end{itemize}

\item There are three exercises below.  No IMRAD this week; that will appear on HW03.

~\\[-2em]

\end{itemize}


\section*{Exercises}

\begin{enumerate} % \itemsep=0pt\parsep=0pt\parskip=0pt

\item Let $y = X\beta + \epsilon$, where $y=(y_1,\ldots, y_n)^T$ is an $n\times 1$ column vector, $X$ is an $n\times (p+1)$ matrix whose first column is all 1's, $\beta=(\beta_0,\ldots,\beta_p)^T$ is a $(p+1)\times 1$ column vector, and $\epsilon=(\epsilon_1,\ldots,\epsilon_n)^T \sim N(0,\sigma^2I)$ is an $n\times 1$ random column vector, following a multivariate Normal distribution with mean vector 0 and variance-covariance matrix $\sigma^2I$, where $I$ is the $n\times n$ identity matrix.

\begin{enumerate}

\item Let 
\[
     U = \left[\begin{array}{c}
               1 \\ 1 \\ \vdots \\ 1
               \end{array}\right]
\]
be an $n\times 1$ column of all 1's.  
\begin{enumerate}
\item Using the definition of the hat matrix $H$, show $HX=X$
\item Using your result in (i), show $HU=U$
\end{enumerate}

\item Use properties of the hat matrix $H=X(X^TX)^{-1}X^T$ to show that the column vector
\[
    y - \hat y = \left[\begin{array}{c}
                       y_1 - \hat y_1 \\
                       y_2 - \hat y_2 \\
                       \vdots \\
                       y_n - \hat y_n
                       \end{array}\right] 
 \]
 can be written as $y - \hat y = (I-H)y$.

\item For any $n\times1$ column vector
\[ 
     a =\left[\begin{array}{c}
     a_1 \\ a_2 \\ \vdots \\ a_n
     \end{array}\right] 
\]
we know that $a^TU=\sum_{i=1}^n a_i$.  Use this fact, together with your results in (a) and (b), to show that 
\[
\sum_{i=1}^n (y_i - \hat y_i) = 0 .
\]  

{\em {\bf Aside:} Hence $y$ and $\hat y$ have the same sample mean $\bar y$, and the residuals vs fitted plot should always be ``balanced'', in some sense, around the horizontal line at $\hat e = 0$.}


\end{enumerate}

\item Let $y = X\beta + \epsilon$, where $y=(y_1,\ldots, y_n)^T$ is an $n\times 1$ column vector, $X$ is an $n\times (p+1)$ matrix whose first column is all 1's, $\beta=(\beta_0,\ldots,\beta_p)^T$ is a $(p+1)\times 1$ column vector, and $\epsilon=(\epsilon_1,\ldots,\epsilon_n)^T \sim N(0,\sigma^2I)$ is an $n\times 1$ random column vector, following a multivariate Normal distribution with mean vector 0 and variance-covariance matrix $\sigma^2I$, where $I$ is the $n\times n$ identity matrix.
\begin{enumerate}\itemsep=0pt\parsep=0pt\parskip=0pt
\item Use properties of the hat matrix $H=X(X^TX)^{-1}X^T$ and the multivariate Normal distribution as discussed in class, to show\footnote{Very similar to problem 1(b).}
\[
			\hat e \sim N(0,(I-H)\sigma^2)
\]
where $\hat e$ is the column vector $\hat e = y - \hat y$.
\item Let $H$ be the hat matrix for the multivariate regression model $y=X\beta + \epsilon$ as in part (a), and let $H_1$ be the hat matrix for the intercept-only model $y=\beta_0  + \epsilon $. 
\begin{enumerate}
\item Show that the fitted values $\hat y $ for the intercept-only model is an $n\times 1$ column vector, all of whose entries are $\bar y$, that is, 
$$
			\hat y  = \left[\begin{array}{c} \hat y _1 \\ \vdots \\ \hat y _n\end{array}\right]
			= \left[\begin{array}{c} \bar y \\ \vdots \\ \bar y\end{array}\right]
			\eqno{(*)}
$$
(where the first ``='' is the definition of $\hat y $ and the second ``='' is what I want you to show).
\item Find a simple expression, in terms of (some or all of) $y$, $I$, $H$ and $H_1$, for the sample covariance
\[
    \mbox{Cov}(y,\hat y) = \frac1n {\sum}_{i=1}^n (y_i - \bar y)(\hat y_i - \bar y).
\]
{\em ({\bf Hint:} We can rewrite $\mbox{Cov}(y,\hat y) = \frac1n (y - \bar y)^T(\hat y - \bar y)$, where $\hat y$ is the column vector of fitted values from $y=X\beta + \epsilon$ and, abusing notation slightly, $\bar y$ is the column vector in $(*)$ above, i.e. the fitted values from the intercept-only model $y = \beta_0  + \epsilon $.)}
\item Continue along the lines of the calculations in part (ii) to show that the sample correlation between $y$ and $\hat y$ can be written as
\[
	    \mbox{Corr}(y,\hat y) = \sqrt{\frac{SS_{reg}}{SST}}
\] 
and hence $R^2$ for the regression model $y = X\beta + \epsilon$ really is the squared correlation between $y$ and $\hat y$:
\[
			R^2 = \mbox{Corr}(y,\hat y)^2~.
\]

\end{enumerate}
\item Show that $\hat e$ and $\hat y$ have sample correlation 0, and hence a scatter plot of $\hat e$ vs $\hat y$ should show no increasing or decreasing overall trend.

\end{enumerate}

{\em {\bf Aside:} It is interesting to note that the results of problem \#1 (c) and problem \#2 (b) and (c) did not depend at all on the normality assumption, or even on whether the model fits the data well or not.  In other words, even if the fit is terrible, it will still be true that the sum of $\hat e$ is zero, $R^2 = [Corr(y,\hat y)]^2$, and the sample correlation between $\hat e$ and $\hat y$ is zero.  The only thing that matters is that the model have an intercept, i.e. the $X$ matrix should have a column of 1's.}

\item{} [Based on Gelman \& Hill. Ch 3, \#1, p. 49] The file {\tt pyth.dat}, in the same folder as this hw,  contains outcome {\tt y} and inputs {\tt x1}, {\tt x2} for 40 data points, with a further 20 points with the inputs but no observed outcome (for this problem we will ignore these last 20 points). Save the file to your working directory and read it into R using the {\tt read.table()} function.

\begin{enumerate}

\item Fit the two models
\begin{eqnarray*}  
	{\bf M1:~} {\tt y} & = & \beta_0 + \beta_1 {\tt x1} + \varepsilon \\
	{\bf M2:~} {\tt y} & = & \beta_0 + \beta_1 {\tt x2} + \varepsilon 
\end{eqnarray*}
Which model provides a better fit for {\tt y}?  Why?

\item Construct new variables {\tt y2 = y$^2$}, {\tt x12 = x1$^2$}, and {\tt x22 = x2$^2$} and fit the models
\begin{eqnarray*}  
	{\bf M3:~} {\tt y2} & = & \beta_0 + \beta_1 {\tt x12} + \varepsilon \\
	{\bf M4:~} {\tt y2} & = & \beta_0 + \beta_1 {\tt x22} + \varepsilon 
\end{eqnarray*}
Compare the fits of these two models to the models in part (a).  Which fits best?  Why?

\item \comment{To fit the model 
\[
	{\tt y2} ~ = ~ \beta_0 + \beta_1 {\tt x1} + \beta_2 {\tt x2} + \varepsilon ~,
\]
we just expand the R modeling language a little bit: \verb|y ~ x1 + x2|. } Fit both of the models
\begin{eqnarray*}
	{\bf M5:~} {\tt ~y} & = & \beta_0 + \beta_1 {\tt x1} + \beta_2 {\tt x2} + \varepsilon \\
	{\bf M6:~} {\tt y2} & = & \beta_0 + \beta_1 {\tt x12}  + \beta_2 {\tt x22} + \varepsilon 
\end{eqnarray*}
Compare these to the earlier models.  Which fits best?  Why?

\item Can you find a simple, recognizable function {\tt x3 = (something involving both x1 and x2)}, so that 
\[
		{\bf M7:~} {\tt y} ~ = ~ \beta_0 + \beta_1 {\tt x3} + \varepsilon
\]
provides a fit comparable to the best fitting models above?  What is going on?

\end{enumerate}

\end{enumerate}

\end{document}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
