% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
%
\documentclass[
]{article}
\usepackage{amsmath,amssymb}
\usepackage{lmodern}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\usepackage[margin=1in]{geometry}
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\usepackage{framed}
\definecolor{shadecolor}{RGB}{248,248,248}
\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\BuiltInTok}[1]{#1}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
\newcommand{\ExtensionTok}[1]{#1}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ImportTok}[1]{#1}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{#1}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\RegionMarkerTok}[1]{#1}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\usepackage{graphicx}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{-\maxdimen} % remove section numbering
\ifLuaTeX
  \usepackage{selnolig}  % disable illegal ligatures
\fi
\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\urlstyle{same} % disable monospaced font for URLs
\hypersetup{
  pdftitle={Homework 02 Solutions},
  hidelinks,
  pdfcreator={LaTeX via pandoc}}

\title{Homework 02 Solutions}
\author{}
\date{\vspace{-2.5em}2022-09-08}

\begin{document}
\maketitle

\def\red#1{\textcolor{red}{#1}}
\def\blue#1{\textcolor{blue}{#1}}
\let\bar\overline
\let\ul\underline

\frenchspacing

\begin{center}
\Large
36-617: Applied Linear Models\\
Fall 2022\\
Solutions
\end{center}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(arm)  }\DocumentationTok{\#\# includes lme4, MASS, Matrix}
\FunctionTok{library}\NormalTok{(ggplot2); }\FunctionTok{theme\_set}\NormalTok{(}\FunctionTok{theme\_bw}\NormalTok{())}
\FunctionTok{library}\NormalTok{(gridExtra) }\DocumentationTok{\#\# to arrange ggplots...}
\end{Highlighting}
\end{Shaded}

\hypertarget{problem-1}{%
\section{Problem 1}\label{problem-1}}

Let \(y = X\beta + \epsilon\), where \(y=(y_1,\ldots, y_n)^T\) is an
\(n\times 1\) column vector, \(X\) is an \(n\times (p+1)\) matrix whose
first column is all 1's, \(\beta=(\beta_0,\ldots,\beta_p)^T\) is a
\((p+1)\times 1\) column vector, and
\(\epsilon=(\epsilon_1,\ldots,\epsilon_n)^T \sim N(0,\sigma^2I)\) is an
\(n\times 1\) random column vector, following a multivariate Normal
distribution with mean vector 0 and variance-covariance matrix
\(\sigma^2I\), where \(I\) is the \(n\times n\) identity matrix.

\hypertarget{problem-1a}{%
\subsection{Problem 1(a)}\label{problem-1a}}

Let \[
     U = \left[\begin{array}{c}
               1 \\ 1 \\ \vdots \\ 1
               \end{array}\right]
\] be an \(n\times 1\) column of all 1's.

\renewcommand{\labelenumi}{\roman{enumi}}

\begin{enumerate}
\item Using the definition of the hat matrix $H$, show $HX=X$
\item Using your result in (i), show $HU=U$
\end{enumerate}

\em

\hypertarget{part-i}{%
\subsubsection{Part (i)}\label{part-i}}

\(HX = [X(X^TX)^{-1}X^T]X = X(X^TX)^{-1}(X^TX) = X\)

\hypertarget{part-ii}{%
\subsubsection{Part (ii)}\label{part-ii}}

Two ways to think about it:

\begin{itemize}
\item \textbf{In words:} Since $HX=X$ we know that all of the columns of $X$ are unchanged when we premultiply $X$ by $H$.  In particular, since the first column of $X$ is $U$, we know $U$ will be unchanged when we premultiply by $H$: $HU=U$.


\item \textbf{In math:}  Let 
\[
      E_1 = \left[\begin{array}{c}
             1 \\ 0 \\ 0 \\ \vdots \\ 0 \end{array}\right] ,
\]
the $n\times 1$ column vector with a 1 in the first row and 0's in the remaining $n-1$ rows.
Since the first column of $X$ is $U$, we know $XE_1 = U$.  Therefore,
\[
     HU = H(XE_1) = (HX)E_1 = XE_1 = U
\]


\end{itemize}

\rm

\hypertarget{problem-1b}{%
\subsection{Problem 1(b)}\label{problem-1b}}

Use properties of the hat matrix \(H=X(X^TX)^{-1}X^T\) to show that the
column vector \[
    y - \hat y = \left[\begin{array}{c}
                       y_1 - \hat y_1 \\
                       y_2 - \hat y_2 \\
                       \vdots \\
                       y_n - \hat y_n
                       \end{array}\right] 
 \] can be written as \(y - \hat y = (I-H)y\).

\em
Since $Hy= \hat y$ and $Iy = y$,
\[
    y - \hat y = Iy - Hy = (I-H)y
\]

\rm

\hypertarget{problem-1c}{%
\subsection{Problem 1(c)}\label{problem-1c}}

For any \(n\times1\) column vector \[ 
     a =\left[\begin{array}{c}
     a_1 \\ a_2 \\ \vdots \\ a_n
     \end{array}\right] 
\] we know that \(a^TU=\sum_{i=1}^n a_i\). Use this fact, together with
your results in (a) and (b), to show that \[
\sum_{i=1}^n (y_i - \hat y_i) = 0 .
\]

\em

\[
     \sum_{i=1}^n (y_i - \hat y_i) = \underbrace{(y - \hat y)^T}_{1\times n}\underbrace{U}_{n\times 1} = \underbrace{[(I-H)y]^T}_{1\times n}\underbrace{U}_{n\times 1} = \underbrace{y^T}_{1\times n}\underbrace{(I-H)}_{n\times n}\underbrace{U}_{n\times 1} = \underbrace{y^T}_{1\times n}\underbrace{(U-U)}_{n\times 1} = \underbrace{Y^T}_{1\times n}\underbrace{0}_{n\times 1} = \underbrace{0}_{1\times 1}
\]

\rm

\textrm{\textbf{Aside:} Hence $y$ and $\hat y$ have the same sample mean $\overline y$, and the residuals vs fitted plot should always be ``balanced'', in some sense, around the horizontal line at $\hat e = 0$.}

\hypertarget{problem-2}{%
\section{Problem 2}\label{problem-2}}

Let \(y = X\beta + \epsilon\), where \(y=(y_1,\ldots, y_n)^T\) is an
\(n\times 1\) column vector, \(X\) is an \(n\times (p+1)\) matrix whose
first column is all 1's, \(\beta=(\beta_0,\ldots,\beta_p)^T\) is a
\((p+1)\times 1\) column vector, and
\(\epsilon=(\epsilon_1,\ldots,\epsilon_n)^T \sim N(0,\sigma^2I)\) is an
\(n\times 1\) random column vector, following a multivariate Normal
distribution with mean vector 0 and variance-covariance matrix
\(\sigma^2I\), where \(I\) is the \(n\times n\) identity matrix.

\hypertarget{problem-2a}{%
\subsection{Problem 2(a)}\label{problem-2a}}

Use properties of the hat matrix \(H=X(X^TX)^{-1}X^T\) and the
multivariate Normal distribution as discussed in class, to
show\footnote{Very similar to problem 1(b).} \[
            \hat e \sim N(0,(I-H)\sigma^2)
\] where \(\hat e\) is the column vector \(\hat e = y - \hat y\).

\em

We will use several facts from lecture. First, let
\(W \sim N(\mu, \Sigma)\) be an arbitrary multivariate normal random
vector and \(A\) a matrix whose second dimension is the same as the
length of \(W\), so that \(AW\) is defined. Then we have \begin{align}
  & AW \sim N(A\mu, A\Sigma A^T) \label{eq:linear_normal}
\end{align} Additionally, we have from lecture that \begin{align}
               & HX\beta = X\beta \label{eq:idempotent1} \\
  & (I - H)(I - H)^T = (I - H)(I - H) = (I - H) \label{eq:idempotent2} \\
                 & y \sim N(X\beta, \sigma^2 I)  \label{eq:y_dist}
\end{align} Let \(\hat{y}\) be the \(n \times 1\) column vector of
fitted values. \begin{align*}
  \hat{e} &= y - \hat{y} \\
          &= y - Hy \\
          &= (I - H)y \\
          &\sim N((I - H)X\beta, (I-H)\sigma^2 I (I-H)^T) \tag{by facts \eqref{eq:linear_normal} and \eqref{eq:y_dist}} \\
          &= N(0, \sigma^2 (I - H)(I - H)^T) \tag{by fact \eqref{eq:idempotent1}} \\
          &= N(0, \sigma^2 (I - H)) \tag{by fact \eqref{eq:idempotent2}}
\end{align*}

\rm

\hypertarget{problem-2b}{%
\subsection{Problem 2(b)}\label{problem-2b}}

Let \(H\) be the hat matrix for the multivariate regression model
\(y=X\beta + \epsilon\) as in part (a), and let \(H_1\) be the hat
matrix for the intercept-only model \(y=\beta_0 + \epsilon\).

\renewcommand{\labelenumi}{\roman{enumi}}

\begin{enumerate}
\item Show that the fitted values $\hat y $ for the intercept-only model is an $n\times 1$ column vector, all of whose entries are $\overline y$, that is, 
$$
            \hat y  = \left[\begin{array}{c} \hat y _1 \\ \vdots \\ \hat y _n\end{array}\right]
            = \left[\begin{array}{c} \overline y \\ \vdots \\ \overline y\end{array}\right]
            \eqno{(*)}
$$
(where the first ``='' is the definition of $\hat y $ and the second ``='' is what I want you to show).

\item Find a simple expression, in terms of (some or all of) $y$, $I$, $H$ and $H_1$, for the sample covariance
\[
    \mbox{Cov}(y,\hat y) = \frac1n {\sum}_{i=1}^n (y_i - \overline y)(\hat y_i - \overline y).
\]
\textit{(\textbf{ Hint:} We can rewrite $\mbox{Cov}(y,\hat y) = \frac1n (y - \overline y)^T(\hat y - \overline y)$, where $\hat y$ is the column vector of fitted values from $y=X\beta + \epsilon$ and, abusing notation slightly, $\overline y$ is the column vector in $(*)$ above, i.e. the fitted values from the intercept-only model $y = \beta_0  + \epsilon $.)}
\item Continue along the lines of the calculations in part (ii) to show that the sample correlation between $y$ and $\hat y$ can be written as
\[
        \mbox{Corr}(y,\hat y) = \sqrt{\frac{SS_{reg}}{SST}}
\] 
and hence $R^2$ for the regression model $y = X\beta + \epsilon$ really is the squared correlation between $y$ and $\hat y$:
\[
            R^2 = \mbox{Corr}(y,\hat y)^2~.
\]

\end{enumerate}

\em

\hypertarget{part-i-1}{%
\subsubsection{Part (i)}\label{part-i-1}}

In the intercept-only model, the design matrix \(X\) is an \(n\times 1\)
matrix in which each entry is 1. This can be used to derive the hat
matrix \(H_1\): \begin{align*}
  X^TX &= [n]_{1\times 1} \tag{the $1\times 1$ matrix whose only entry is $n$} \\
  \implies (X^TX)^{-1} &= [1/n]_{1\times 1} \tag{the $1\times 1$ matrix whose only entry is $1/n$} \\
  \implies X(X^TX)^{-1}X^T &= \frac{1}{n}XX^T \\
                           &= \frac{1}{n} \begin{bmatrix}
                           1 & \ldots & 1 \\
                           \vdots & \ddots & \vdots \\
                           1 & \ldots & 1 \end{bmatrix}_{n\times n} \\
                           &= H_1 \tag{by the definition of the hat matrix}
\end{align*} For any \(i\) in \(1, 2, \ldots, n\), we derive
\(\hat{y}_i\) by multiplying the \(i\)th row of \(H_1\) by the vector
\(y\). In other words, for the intercept-only model,
\(\hat{y}_i = \frac{1}{n}\sum_{i=1}^n y_i = \overline{y}\) for all
\(i\).

\hypertarget{part-ii-1}{%
\subsubsection{Part (ii)}\label{part-ii-1}}

To avoid using \(\overline{y}\) to denote both a scalar and a vector,
let's immediately use the fact that \(H_1y\) is equal to the
\(n\times 1\) vector
\(\begin{bmatrix} \overline{y} & \overline{y} & \ldots & \overline{y}\end{bmatrix}^T\),
as shown in the previous problem. Rewriting the covariance in vector
form, per the problem suggestion, we have: \begin{align*}
  \mathop{\mathrm{Cov}}(y, \hat{y}) &= \frac1n {\sum}_{i=1}^n (y_i - \overline y)(\hat y_i - \overline y) \\
                   &= \frac{1}{n}(y - H_1y)^T(\hat{y} - H_1y) \\
                   &= \frac{1}{n}(y - H_1y)^T(Hy - H_1y) \\
                   &= \frac{1}{n}[(I - H_1)y]^T(H - H_1)y \\
                   &= \frac{1}{n}y^T(I - H_1)^T(H - H_1)y \tag{since $(AB)^T = B^T A^T$} \\
                   &= \frac{1}{n}y^T(H - H_1 - H_1^TH + H_1^TH_1)y \\
                   &= \frac{1}{n}y^T(H - H_1H)y \tag{since $H_1^T = H_1$ and $H_1H_1 = H_1$} \\
                   &= \frac{1}{n}y^T(H - H_1)y \tag{since $H_1H = HH_1 = H_1$}
\end{align*}

\hypertarget{part-iii}{%
\subsubsection{Part (iii)}\label{part-iii}}

Recall that the sample correlation is the standardized sample
covariance, defined as \begin{align*}
  \\widehat{\mathop{\mathrm{Corr}}}(y, \hat{y}) &= \frac{\hat{\mathop{\mathrm{Cov}}}(y, \hat{y})}{\sqrt{ \hat{\sigma}_y^2 \hat{\sigma}_{\hat{y}}^2 }} \\
  &= \frac{\sum_{i=1}^n (y_i - \overline{y})(\hat{y}_i - \overline{y})} {\sqrt{\sum_{i=1}^n (y_i - \overline{y})^2 \sum_{i=1}^n (\hat{y}_i - \overline{y})^2}} \\
  &= \frac{(y-H_1y)^T(\hat{y}-H_1y)}{\sqrt{(y-H_1y)^T(y-H_1y)(\hat{y}-H_1y)^T(\hat{y}-H_1y)}} \\
  &= \frac{y^T(H-H_1)y}{\sqrt{y^T(I-H_1)^T(I-H_1)y y^T(H-H_1)^T(H-H_1)y}} \tag{using the previous problem} \\
  &= \frac{\sqrt{y^T(H-H_1)y}}{\sqrt{y^T(I-H_1)^T(I-H_1)y}} \\
  &= \frac{\sqrt{y^T(H-H_1)y}}{\sqrt{y^T(I-H_1)y}} \tag{using a version of fact \eqref{eq:idempotent2}} \\
  &= \sqrt{\frac{SS_{reg}}{SST}} \\
  &= \sqrt{R^2} \tag{by the definition of $R^2$}
\end{align*} And therefore,
\(R^2 = \widehat{\mathop{\mathrm{Corr}}}(y, \hat{y})^2\).

\rm

\hypertarget{problem-2c}{%
\subsection{Problem 2(c)}\label{problem-2c}}

Show that \(\hat e\) and \(\hat y\) have sample correlation 0, and hence
a scatter plot of \(\hat e\) vs \(\hat y\) should show no increasing or
decreasing overall trend.

\em

The sample correlation is 0 iff the sample covariance is 0, so let's
just concern ourselves with the sample covariance and not worry about
the denominator in the correlation.

Recall that the residuals sum to 0 by construction, which also means
that their sample mean is 0. Hence: \begin{align*}
  \hat{\mathop{\mathrm{Cov}}}(\hat{e}, \hat{y}) &= \frac{1}{n}\sum_{i=1}^n (\hat{e_i} - 0)(\hat{y_i} - \overline{y}) \\
  &= \frac{1}{n}(\hat{e})^T(\hat{y} - \overline{y}) \\
  &= \frac{1}{n}(y - \hat{y})^T(\hat{y} - \overline{y}) \\
  &= \frac{1}{n}[(I-H)y]^T(H-H_1)y \\
  &= \frac{1}{n}y^T(I-H)(H-H_1)y \tag{since $(I-H)^T = (I-H)$} \\
  &= \frac{1}{n}y^T(H-H_1 - HH + HH_1)y \\
  &= \frac{1}{n}y^T(H - H_1 - H + H_1)y \tag{since $HH_1 = H_1$} \\
  &= 0
\end{align*}

\rm

\textrm{\textbf{Aside:} It is interesting to note that the results of problem \#1 (c) and problem \#2 (b) and (c) did not depend at all on the normality assumption, or even on whether the model fits the data well or not.  In other words, even if the fit is terrible, it will still be true that the sum of $\hat e$ is zero, $R^2 = [Corr(y,\hat y)]^2$, and the sample correlation between $\hat e$ and $\hat y$ is zero.  The only thing that matters is that the model have an intercept, i.e. the $X$ matrix should have a column of 1's.}

\hypertarget{problem-3}{%
\subsection{Problem 3}\label{problem-3}}

{[}Based on Gelman \& Hill. Ch 3, \#1, p.~49{]} The file
\texttt{ pyth.dat}, in the same folder as this hw, contains outcome
\texttt{ y} and inputs \texttt{ x1}, \texttt{ x2} for 40 data points,
with a further 20 points with the inputs but no observed outcome (for
this problem we will ignore these last 20 points). Save the file to your
working directory and read it into R using the \texttt{ read.table()}
function.

\hypertarget{problem-3a}{%
\subsection{Problem 3(a)}\label{problem-3a}}

Fit the two models \begin{eqnarray*}  
    \textbf{ M1:~} \texttt{ y} & = & \beta_0 + \beta_1 \texttt{ x1} + \varepsilon \\
    \textbf{ M2:~} \texttt{ y} & = & \beta_0 + \beta_1 \texttt{ x2} + \varepsilon 
\end{eqnarray*} Which model provides a better fit for \texttt{ y}? Why?

\em

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{gh.data }\OtherTok{\textless{}{-}} \FunctionTok{read.table}\NormalTok{(}\StringTok{"pyth.dat"}\NormalTok{,}\AttributeTok{header=}\NormalTok{T)}
\NormalTok{gh.data }\OtherTok{\textless{}{-}}\NormalTok{ gh.data[}\FunctionTok{apply}\NormalTok{(gh.data,}\DecValTok{1}\NormalTok{,}\ControlFlowTok{function}\NormalTok{(x) \{}\SpecialCharTok{!}\FunctionTok{any}\NormalTok{(}\FunctionTok{is.na}\NormalTok{(x))\}),]}
\FunctionTok{str}\NormalTok{(gh.data)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 'data.frame':    40 obs. of  3 variables:
##  $ y : num  15.68 6.18 18.1 9.07 17.97 ...
##  $ x1: num  6.87 4.4 0.43 2.73 3.25 5.3 7.08 9.73 4.51 6.4 ...
##  $ x2: num  14.09 4.35 18.09 8.65 17.68 ...
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{M1 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y }\SpecialCharTok{\textasciitilde{}}\NormalTok{ x1, }\AttributeTok{data=}\NormalTok{gh.data)}
\NormalTok{M2 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y }\SpecialCharTok{\textasciitilde{}}\NormalTok{ x2, }\AttributeTok{data=}\NormalTok{gh.data)}

\FunctionTok{summary}\NormalTok{(M1)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y ~ x1, data = gh.data)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -7.7409 -4.5056  0.7114  4.3739  7.7547 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  10.0633     1.5526   6.481 1.25e-07 ***
## x1            0.6559     0.2499   2.625   0.0124 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 4.921 on 38 degrees of freedom
## Multiple R-squared:  0.1535, Adjusted R-squared:  0.1312 
## F-statistic:  6.89 on 1 and 38 DF,  p-value: 0.01242
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(M2)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y ~ x2, data = gh.data)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -3.1751 -1.2352 -0.1867  1.0899  5.3755 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  3.78532    0.66037   5.732 1.33e-06 ***
## x2           0.83223    0.05017  16.589  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 1.863 on 38 degrees of freedom
## Multiple R-squared:  0.8787, Adjusted R-squared:  0.8755 
## F-statistic: 275.2 on 1 and 38 DF,  p-value: < 2.2e-16
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\FunctionTok{par}\NormalTok{(}\AttributeTok{mfrow=}\FunctionTok{c}\NormalTok{(}\DecValTok{2}\NormalTok{,}\DecValTok{2}\NormalTok{))}
\FunctionTok{plot}\NormalTok{(M1)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-2-1} \end{center}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(M2)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-2-2} \end{center}

The \(R^2\) is much lower for M1 (0.1535) than for M2 (0.8787). Neither
set of residual diagnostic plots looks great: the residual vs fitted and
scale-location plots somewhat favor M1, and the QQ plots somewhat favor
M2. The Cook's distances are a bit better for M1 also. For \(R^2\) and
normality of residuals, I prefer M2.

\rm

\hypertarget{problem-3b}{%
\subsection{Problem 3(b)}\label{problem-3b}}

Construct new variables \texttt{ y2 = y$^2$}, \texttt{ x12 = x1$^2$},
and \texttt{ x22 = x2$^2$} and fit the models \begin{eqnarray*}  
    \textbf{ M3:~} \texttt{ y2} & = & \beta_0 + \beta_1 \texttt{ x12} + \varepsilon \\
    \textbf{ M4:~} \texttt{ y2} & = & \beta_0 + \beta_1 \texttt{ x22} + \varepsilon 
\end{eqnarray*} Compare the fits of these two models to the models in
part (a). Which fits best? Why?

\em

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{attach}\NormalTok{(gh.data)}
\NormalTok{y2 }\OtherTok{\textless{}{-}}\NormalTok{ y}\SpecialCharTok{\^{}}\DecValTok{2}
\NormalTok{x12 }\OtherTok{\textless{}{-}}\NormalTok{ x1}\SpecialCharTok{\^{}}\DecValTok{2}
\NormalTok{x22 }\OtherTok{\textless{}{-}}\NormalTok{ x2}\SpecialCharTok{\^{}}\DecValTok{2}
\FunctionTok{detach}\NormalTok{()}
\NormalTok{M3 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y2 }\SpecialCharTok{\textasciitilde{}}\NormalTok{ x12,}\AttributeTok{data=}\NormalTok{gh.data)}
\NormalTok{M4 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y2 }\SpecialCharTok{\textasciitilde{}}\NormalTok{ x22,}\AttributeTok{data=}\NormalTok{gh.data)}

\FunctionTok{summary}\NormalTok{(M3)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y2 ~ x12, data = gh.data)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -189.324 -125.674    4.988  131.052  214.089 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 161.7831    31.7873   5.090    1e-05 ***
## x12           1.2971     0.6242   2.078   0.0445 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 131.1 on 38 degrees of freedom
## Multiple R-squared:  0.102,  Adjusted R-squared:  0.07841 
## F-statistic: 4.318 on 1 and 38 DF,  p-value: 0.04452
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(M4)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y2 ~ x22, data = gh.data)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -41.280 -31.224  -7.463  25.422  59.571 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  35.1583     9.0306   3.893 0.000387 ***
## x22           1.0198     0.0419  24.338  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 33.96 on 38 degrees of freedom
## Multiple R-squared:  0.9397, Adjusted R-squared:  0.9381 
## F-statistic: 592.3 on 1 and 38 DF,  p-value: < 2.2e-16
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{par}\NormalTok{(}\AttributeTok{mfrow=}\FunctionTok{c}\NormalTok{(}\DecValTok{2}\NormalTok{,}\DecValTok{2}\NormalTok{))}
\FunctionTok{plot}\NormalTok{(M3)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-3-1} \end{center}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(M4)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-3-2} \end{center}

Model M4 has the highest \(R^2\) (0.9397), and has residuals vs fitted
and scale-location plots that are at least as good as any of the others;
on the other hand, we seem to be losing normality of the residuals.
Nevertheless I prefer M4 so far.

\rm

\hypertarget{problem-3c}{%
\subsection{problem 3(c)}\label{problem-3c}}

Fit both of the models \begin{eqnarray*}
    \textbf{ M5:~} \texttt{ ~y} & = & \beta_0 + \beta_1 \texttt{ x1} + \beta_2 \texttt{ x2} + \varepsilon \\
    \textbf{ M6:~} \texttt{ y2} & = & \beta_0 + \beta_1 \texttt{ x12}  + \beta_2 \texttt{ x22} + \varepsilon 
\end{eqnarray*} Compare these to the earlier models. Which fits best?
Why?

\em

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{M5 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y }\SpecialCharTok{\textasciitilde{}}\NormalTok{ x1 }\SpecialCharTok{+}\NormalTok{ x2, }\AttributeTok{data=}\NormalTok{gh.data)}
\NormalTok{M6 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y2 }\SpecialCharTok{\textasciitilde{}}\NormalTok{ x12 }\SpecialCharTok{+}\NormalTok{ x22, }\AttributeTok{data=}\NormalTok{gh.data)}

\FunctionTok{summary}\NormalTok{(M5)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y ~ x1 + x2, data = gh.data)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -0.9585 -0.5865 -0.3356  0.3973  2.8548 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.31513    0.38769   3.392  0.00166 ** 
## x1           0.51481    0.04590  11.216 1.84e-13 ***
## x2           0.80692    0.02434  33.148  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.9 on 37 degrees of freedom
## Multiple R-squared:  0.9724, Adjusted R-squared:  0.9709 
## F-statistic: 652.4 on 2 and 37 DF,  p-value: < 2.2e-16
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(M6)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y2 ~ x12 + x22, data = gh.data)
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -0.26020 -0.05391 -0.00396  0.06367  0.35990 
## 
## Coefficients:
##              Estimate Std. Error  t value Pr(>|t|)    
## (Intercept) 0.0026691  0.0422669    0.063     0.95    
## x12         0.9999672  0.0006419 1557.713   <2e-16 ***
## x22         0.9998685  0.0001663 6011.909   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.1344 on 37 degrees of freedom
## Multiple R-squared:      1,  Adjusted R-squared:      1 
## F-statistic: 2.013e+07 on 2 and 37 DF,  p-value: < 2.2e-16
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{par}\NormalTok{(}\AttributeTok{mfrow=}\FunctionTok{c}\NormalTok{(}\DecValTok{2}\NormalTok{,}\DecValTok{2}\NormalTok{))}
\FunctionTok{plot}\NormalTok{(M5)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-4-1} \end{center}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(M6)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-4-2} \end{center}

Putting both x1 and x2 in the model for y really improved the model: M5
has an \(R^2\) of 0.9724, and both predictors are significant (have
coefficient estimates significantly different from zero). However the
residual diagnostic plots don't look great; in particular it seems like
the residuals have a lot of right skew, and leverage seems tio increase
with the size of the standardized resduals.

M6 is really winning, though: \(R^2=1\), and the QQ plot shows good
agreement between the residuals and the normal distribution. There seems
to be some evidence for non-constant variance though: the residuals vs
fitted plot fans out as the fitted valuess increase, and the
scale-location plot tells a similar story. On the other hand, only one
data point seems to have a concerning Cook's distance.

Based on all of this I like M6 best. Looking at the estimated
coefficients for M6, I notice something interesting: \(\hat\beta_0\) is
indistinguishable from 0, and both \(\hat\beta_1\) and \(\hat\beta_2\)
equal 1, to at least two decimal places (even if we compute the 95\%
CI's!).

\rm

\hypertarget{problem-3d}{%
\subsection{Problem 3(d)}\label{problem-3d}}

Can you find a simple, recognizable function
\texttt{ x3 = (something involving both x1 and x2)}, so that \[
        \textbf{ M7:~} \texttt{ y} ~ = ~ \beta_0 + \beta_1 \texttt{ x3} + \varepsilon
\] provides a fit comparable to the best fitting models above? What is
going on?

\em

In problem 3(c) we saw that the model M6 was very nearly \[
    y^2 = x1^2 + x2^2 + \varepsilon
\] If we ignore \(\varepsilon\), take square roots, and put
\(\varepsilon\) and some ``unknown'\,' regression coefficients back in,
we get a model like \[
    y = \beta_0 + \beta_1\sqrt{x1^2 + x2^2} + \varepsilon
\] i.e., \(y\) is the distance to the origin from some points
\((x1,x2)\) in Cartesian space.

Let's try fitting this model:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{M7 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(y }\SpecialCharTok{\textasciitilde{}} \FunctionTok{I}\NormalTok{(}\FunctionTok{sqrt}\NormalTok{(x1}\SpecialCharTok{\^{}}\DecValTok{2} \SpecialCharTok{+}\NormalTok{ x2}\SpecialCharTok{\^{}}\DecValTok{2}\NormalTok{)),}\AttributeTok{data=}\NormalTok{gh.data)}
\FunctionTok{summary}\NormalTok{(M7)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = y ~ I(sqrt(x1^2 + x2^2)), data = gh.data)
## 
## Residuals:
##        Min         1Q     Median         3Q        Max 
## -0.0083283 -0.0027000 -0.0007907  0.0031643  0.0089809 
## 
## Coefficients:
##                       Estimate Std. Error  t value Pr(>|t|)    
## (Intercept)          0.0018422  0.0019159    0.962    0.342    
## I(sqrt(x1^2 + x2^2)) 0.9998313  0.0001316 7596.431   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.00434 on 38 degrees of freedom
## Multiple R-squared:      1,  Adjusted R-squared:      1 
## F-statistic: 5.771e+07 on 1 and 38 DF,  p-value: < 2.2e-16
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{par}\NormalTok{(}\AttributeTok{mfrow=}\FunctionTok{c}\NormalTok{(}\DecValTok{2}\NormalTok{,}\DecValTok{2}\NormalTok{))}
\FunctionTok{plot}\NormalTok{(M7)}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics{hw02-solutions_files/figure-latex/unnamed-chunk-5-1} \end{center}

This seems to confirm our suspicions! \(R^2=1\), the estimated
regression coefficients are essentially \(\hat\beta_0 = 0\) and
\(\hat\beta_1=1\), and the diagnostic plots looks great:

\begin{itemize}
\item The residual vs fitted plot shows little vertical structure.
\item The QQ plot shows good adherence to normality.
\item The scale-location plot is consistent with constant-variance residuals.
\item None of the data points has Cook's distance above 0.5.
\end{itemize}

\rm

\end{document}
