% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
%
\documentclass[
]{article}
\usepackage{amsmath,amssymb}
\usepackage{lmodern}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
\hypersetup{
  pdftitle={Homework 10 Solutions},
  hidelinks,
  pdfcreator={LaTeX via pandoc}}
\urlstyle{same} % disable monospaced font for URLs
\usepackage[margin=1in]{geometry}
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\usepackage{framed}
\definecolor{shadecolor}{RGB}{248,248,248}
\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\BuiltInTok}[1]{#1}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
\newcommand{\ExtensionTok}[1]{#1}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ImportTok}[1]{#1}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{#1}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\RegionMarkerTok}[1]{#1}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\usepackage{graphicx}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{-\maxdimen} % remove section numbering
\usepackage{amsmath}
\usepackage{amsthm}
\DeclareMathOperator{\Cov}{Cov}
\DeclareMathOperator{\Corr}{Corr}
\DeclareMathOperator{\Var}{Var}
\DeclareMathOperator{\upmodels}{\perp \!\!\! \perp}
\ifLuaTeX
  \usepackage{selnolig}  % disable illegal ligatures
\fi

\title{Homework 10 Solutions}
\author{}
\date{\vspace{-2.5em}11/17/2021}

\begin{document}
\maketitle

\hypertarget{problem-1}{%
\section{Problem 1}\label{problem-1}}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(arm)}
\FunctionTok{library}\NormalTok{(lme4)}
\FunctionTok{library}\NormalTok{(latex2exp) }\CommentTok{\# for nice plot labeling (using latex notation)}

\NormalTok{cdi }\OtherTok{\textless{}{-}} \FunctionTok{read.table}\NormalTok{(}\AttributeTok{file =} \StringTok{"../cdi.dat"}\NormalTok{)}

\NormalTok{cdi}\SpecialCharTok{$}\NormalTok{pci }\OtherTok{\textless{}{-}} \FunctionTok{rescale}\NormalTok{(}\DecValTok{2}\SpecialCharTok{*}\NormalTok{cdi}\SpecialCharTok{$}\NormalTok{per.cap.income) }
\NormalTok{cdi}\SpecialCharTok{$}\NormalTok{phg }\OtherTok{\textless{}{-}} \FunctionTok{rescale}\NormalTok{(}\DecValTok{2}\SpecialCharTok{*}\NormalTok{cdi}\SpecialCharTok{$}\NormalTok{pct.hs.grad)}
\end{Highlighting}
\end{Shaded}

\hypertarget{a}{%
\subsection{(a)}\label{a}}

See Figure \ref{f:fig1a}.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# pooled regression}
\NormalTok{lm\_all }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(pci }\SpecialCharTok{\textasciitilde{}}\NormalTok{ phg, }\AttributeTok{data =}\NormalTok{ cdi)}

\NormalTok{lm\_all\_rep }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{intercept =}\NormalTok{ lm\_all[[}\StringTok{"coefficients"}\NormalTok{]][}\DecValTok{1}\NormalTok{],}
                         \AttributeTok{slope =}\NormalTok{ lm\_all[[}\StringTok{"coefficients"}\NormalTok{]][}\DecValTok{2}\NormalTok{],}
                         \AttributeTok{state =} \FunctionTok{unique}\NormalTok{(cdi}\SpecialCharTok{$}\NormalTok{state))}

\CommentTok{\# unpooled regression}
\NormalTok{state\_lm\_func }\OtherTok{\textless{}{-}} \ControlFlowTok{function}\NormalTok{(df)\{}
\NormalTok{    mod }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(pci }\SpecialCharTok{\textasciitilde{}}\NormalTok{ phg, }\AttributeTok{data =}\NormalTok{ df)}
    \FunctionTok{return}\NormalTok{(}\FunctionTok{data.frame}\NormalTok{(}\AttributeTok{intercept =}\NormalTok{ mod[[}\StringTok{"coefficients"}\NormalTok{]][}\DecValTok{1}\NormalTok{], }
                      \AttributeTok{slope =}\NormalTok{ mod[[}\StringTok{"coefficients"}\NormalTok{]][}\DecValTok{2}\NormalTok{]))}
\NormalTok{\}}

\NormalTok{state\_lm\_df }\OtherTok{\textless{}{-}}\NormalTok{ cdi }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{group\_by}\NormalTok{(state) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{do}\NormalTok{(., }\FunctionTok{state\_lm\_func}\NormalTok{(.))}

\DocumentationTok{\#\# unpooled regression version II}
\NormalTok{lm\_unpooled }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(pci }\SpecialCharTok{\textasciitilde{}}\NormalTok{ state }\SpecialCharTok{+}\NormalTok{ phg}\SpecialCharTok{:}\NormalTok{state }\SpecialCharTok{{-}} \DecValTok{1}\NormalTok{, }\AttributeTok{data =}\NormalTok{ cdi) }\CommentTok{\# single model}

\NormalTok{coef\_unpooled }\OtherTok{\textless{}{-}} \FunctionTok{coef}\NormalTok{(lm\_unpooled)}
\NormalTok{state\_loc }\OtherTok{\textless{}{-}} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(coef\_unpooled), }\DecValTok{6}\NormalTok{,}\DecValTok{7}\NormalTok{)}
\NormalTok{slope\_loc }\OtherTok{\textless{}{-}} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(coef\_unpooled), }\DecValTok{9}\NormalTok{,}\DecValTok{11}\NormalTok{)}

\NormalTok{intercepts }\OtherTok{\textless{}{-}}\NormalTok{ coef\_unpooled[slope\_loc }\SpecialCharTok{==} \StringTok{""}\NormalTok{] }
\NormalTok{intercepts\_df }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{state =} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(intercepts), }\DecValTok{6}\NormalTok{,}\DecValTok{7}\NormalTok{),}
                            \AttributeTok{intercept =}\NormalTok{ intercepts)}
\NormalTok{slopes }\OtherTok{\textless{}{-}}\NormalTok{ coef\_unpooled[slope\_loc }\SpecialCharTok{==} \StringTok{"phg"}\NormalTok{]}
\NormalTok{slopes\_df }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{state =} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(slopes), }\DecValTok{6}\NormalTok{,}\DecValTok{7}\NormalTok{),}
                        \AttributeTok{slope =}\NormalTok{ slopes)}

\NormalTok{state\_lm\_df2 }\OtherTok{\textless{}{-}}\NormalTok{ intercepts\_df }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{left\_join}\NormalTok{(slopes\_df, }\AttributeTok{by =} \StringTok{\textquotesingle{}state\textquotesingle{}}\NormalTok{)}

\NormalTok{testthat}\SpecialCharTok{::}\FunctionTok{expect\_equivalent}\NormalTok{(state\_lm\_df, state\_lm\_df2) }\CommentTok{\# no error {-} so true}

\DocumentationTok{\#\# unpooled regression version III}
\NormalTok{lm\_unpooled2 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(pci }\SpecialCharTok{\textasciitilde{}}\NormalTok{ state}\SpecialCharTok{*}\NormalTok{phg }\SpecialCharTok{{-}} \DecValTok{1} \SpecialCharTok{{-}}\NormalTok{ phg, }\AttributeTok{data =}\NormalTok{ cdi) }\CommentTok{\# single model}

\NormalTok{coef\_unpooled2 }\OtherTok{\textless{}{-}} \FunctionTok{coef}\NormalTok{(lm\_unpooled2)}
\NormalTok{state\_loc2 }\OtherTok{\textless{}{-}} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(coef\_unpooled2), }\DecValTok{6}\NormalTok{,}\DecValTok{7}\NormalTok{)}
\NormalTok{slope\_loc2 }\OtherTok{\textless{}{-}} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(coef\_unpooled2), }\DecValTok{9}\NormalTok{,}\DecValTok{11}\NormalTok{)}

\NormalTok{intercepts2 }\OtherTok{\textless{}{-}}\NormalTok{ coef\_unpooled2[slope\_loc2 }\SpecialCharTok{==} \StringTok{""}\NormalTok{] }
\NormalTok{intercepts\_df2 }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{state =} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(intercepts2), }\DecValTok{6}\NormalTok{,}\DecValTok{7}\NormalTok{),}
                             \AttributeTok{intercept =}\NormalTok{ intercepts2)}
\NormalTok{slopes2 }\OtherTok{\textless{}{-}}\NormalTok{ coef\_unpooled2[slope\_loc2 }\SpecialCharTok{==} \StringTok{"phg"}\NormalTok{]}
\NormalTok{slopes\_df2 }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{state =} \FunctionTok{substring}\NormalTok{(}\FunctionTok{names}\NormalTok{(slopes2), }\DecValTok{6}\NormalTok{,}\DecValTok{7}\NormalTok{),}
                         \AttributeTok{slope =}\NormalTok{ slopes2)}

\NormalTok{state\_lm\_df3 }\OtherTok{\textless{}{-}}\NormalTok{ intercepts\_df2 }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{left\_join}\NormalTok{(slopes\_df2, }\AttributeTok{by =} \StringTok{\textquotesingle{}state\textquotesingle{}}\NormalTok{)}

\NormalTok{testthat}\SpecialCharTok{::}\FunctionTok{expect\_equivalent}\NormalTok{(state\_lm\_df, state\_lm\_df3) }\CommentTok{\# no error {-} so true}


\CommentTok{\# multilevel model}
\NormalTok{multilevel\_lmer }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\AttributeTok{formula =}\NormalTok{ pci }\SpecialCharTok{\textasciitilde{}} \DecValTok{1} \SpecialCharTok{+}\NormalTok{ phg }\SpecialCharTok{+}\NormalTok{ (}\DecValTok{1} \SpecialCharTok{|}\NormalTok{ state) }\SpecialCharTok{+}\NormalTok{ (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ phg }\SpecialCharTok{|}\NormalTok{ state), }
                        \AttributeTok{data =}\NormalTok{ cdi)}

\NormalTok{multilevel\_lmer\_df }\OtherTok{\textless{}{-}} \FunctionTok{coef}\NormalTok{(multilevel\_lmer)}\SpecialCharTok{$}\NormalTok{state }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{  tibble}\SpecialCharTok{::}\FunctionTok{rownames\_to\_column}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{rename}\NormalTok{(}\AttributeTok{state =} \StringTok{"rowname"}\NormalTok{,}
         \AttributeTok{intercept =} \StringTok{"(Intercept)"}\NormalTok{,}
         \AttributeTok{slope =} \StringTok{"phg"}\NormalTok{)}

\CommentTok{\# multilevel model version II}
\NormalTok{betas }\OtherTok{\textless{}{-}} \FunctionTok{fixef}\NormalTok{(multilevel\_lmer)}
\NormalTok{etas }\OtherTok{\textless{}{-}} \FunctionTok{ranef}\NormalTok{(multilevel\_lmer)}\SpecialCharTok{$}\NormalTok{state}

\NormalTok{combo\_multilevel\_lmer\_df }\OtherTok{\textless{}{-}}\NormalTok{ etas }\SpecialCharTok{+} 
  \FunctionTok{matrix}\NormalTok{(}\FunctionTok{rep}\NormalTok{(betas, }\AttributeTok{each =} \FunctionTok{nrow}\NormalTok{(etas)), }\AttributeTok{nrow =} \FunctionTok{nrow}\NormalTok{(etas))}

\NormalTok{testthat}\SpecialCharTok{::}\FunctionTok{expect\_equivalent}\NormalTok{(combo\_multilevel\_lmer\_df,}
                            \FunctionTok{coef}\NormalTok{(multilevel\_lmer)}\SpecialCharTok{$}\NormalTok{state) }\CommentTok{\# doesn\textquotesingle{}t error if true}

\CommentTok{\# combined}
\NormalTok{combined }\OtherTok{\textless{}{-}} \FunctionTok{rbind}\NormalTok{(}\FunctionTok{mutate}\NormalTok{(lm\_all\_rep[, }\FunctionTok{c}\NormalTok{(}\StringTok{"state"}\NormalTok{, }\StringTok{"intercept"}\NormalTok{, }\StringTok{"slope"}\NormalTok{)], }
                         \AttributeTok{id =} \StringTok{"pooled regression"}\NormalTok{),}
                  \FunctionTok{mutate}\NormalTok{(state\_lm\_df[, }\FunctionTok{c}\NormalTok{(}\StringTok{"state"}\NormalTok{, }\StringTok{"intercept"}\NormalTok{, }\StringTok{"slope"}\NormalTok{)], }
                         \AttributeTok{id =} \StringTok{"unpooled regression"}\NormalTok{),}
                  \FunctionTok{mutate}\NormalTok{(multilevel\_lmer\_df[, }\FunctionTok{c}\NormalTok{(}\StringTok{"state"}\NormalTok{, }\StringTok{"intercept"}\NormalTok{, }\StringTok{"slope"}\NormalTok{)],}
                         \AttributeTok{id =} \StringTok{"multilevel model"}\NormalTok{)) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{id =} \FunctionTok{factor}\NormalTok{(id, }\AttributeTok{levels =} \FunctionTok{c}\NormalTok{(}\StringTok{"pooled regression"}\NormalTok{,}
                                    \StringTok{"unpooled regression"}\NormalTok{,}
                                    \StringTok{"multilevel model"}\NormalTok{)))}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(cdi) }\SpecialCharTok{+}
  \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ phg, }\AttributeTok{y=}\NormalTok{ pci)) }\SpecialCharTok{+}
  \FunctionTok{geom\_abline}\NormalTok{(}\AttributeTok{data =}\NormalTok{ combined,}
              \FunctionTok{aes}\NormalTok{(}\AttributeTok{slope =}\NormalTok{ slope, }\AttributeTok{intercept=}\NormalTok{ intercept, }\AttributeTok{group =}\NormalTok{ state,}
                  \AttributeTok{color =}\NormalTok{ id))}\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{ state) }\SpecialCharTok{+}
  \FunctionTok{theme\_bw}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{color =} \StringTok{"model type"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{figure}
\centering
\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-3-1.pdf}
\caption{Problem 1a: Different linear models to compare per capital
income to percent high school graduates \label{f:fig1a}.}
\end{figure}

\hypertarget{b}{%
\subsection{(b)}\label{b}}

See figure \ref{f:fig1b}.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# multilevel model}
\NormalTok{multilevel\_res\_fit }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{fitted =} \FunctionTok{fitted}\NormalTok{(multilevel\_lmer), }
                                 \AttributeTok{residual =} \FunctionTok{resid}\NormalTok{(multilevel\_lmer), }
                                 \AttributeTok{state =}\NormalTok{ cdi}\SpecialCharTok{$}\NormalTok{state)}

\CommentTok{\# multilevel model, part II}
\FunctionTok{source}\NormalTok{(}\StringTok{"../residual{-}functions.r"}\NormalTok{)}
\NormalTok{multilevel\_res\_fit2 }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{fitted =} \FunctionTok{yhat.cond}\NormalTok{(multilevel\_lmer),}
                                  \AttributeTok{residual =} \FunctionTok{r.cond}\NormalTok{(multilevel\_lmer),}
                                  \AttributeTok{state =}\NormalTok{ cdi}\SpecialCharTok{$}\NormalTok{state)}

\NormalTok{testthat}\SpecialCharTok{::}\FunctionTok{expect\_equivalent}\NormalTok{(multilevel\_res\_fit, multilevel\_res\_fit2) }\CommentTok{\# no error if true}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{multilevel\_res\_fit }\SpecialCharTok{\%\textgreater{}\%}
\FunctionTok{ggplot}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ fitted, }\AttributeTok{y =}\NormalTok{ residual)) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{ state) }\SpecialCharTok{+}
  \FunctionTok{geom\_hline}\NormalTok{(}\AttributeTok{yintercept =} \DecValTok{0}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{theme\_bw}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{y =} \StringTok{"conditional residual"}\NormalTok{,}
       \AttributeTok{x =} \FunctionTok{TeX}\NormalTok{(}\StringTok{"conditional $}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{hat\{y\}$"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

\begin{figure}
\centering
\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-5-1.pdf}
\caption{Problem 1b: Examinig the conditional residual vs conditional
\(\hat{y}\) for multilevel model \label{f:fig1b}.}
\end{figure}

\clearpage

\hypertarget{problem-2}{%
\section{Problem 2}\label{problem-2}}

NOTES:

\begin{enumerate}
\def\labelenumi{(\arabic{enumi})}
\item
  Instead of giving ``full'' solutions for \#2, I'm just going to
  provide some suggestive code\ldots{} Mostly I am not providing
  interpretations-- you will need to do that in your IDMRAD paper for
  the college, after you are satisfied with your technical appendix!
\item
  There are many pages here because I couldn't suppress some of the
  output from fitLMER.fnc() from library(LMERconveniencefunctions).
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(lme4)}
\FunctionTok{library}\NormalTok{(arm)}
\FunctionTok{library}\NormalTok{(ggplot2)}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# Read the data in wide and tall formats...}
\NormalTok{ratings }\OtherTok{\textless{}{-}} \FunctionTok{read.csv}\NormalTok{(}\StringTok{"ratings.csv"}\NormalTok{,}\AttributeTok{header=}\NormalTok{T)}
\NormalTok{tall }\OtherTok{\textless{}{-}}  \FunctionTok{read.csv}\NormalTok{(}\StringTok{"tall.csv"}\NormalTok{,}\AttributeTok{header=}\NormalTok{T)}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# Some preliminary exploration of the data with View(), summary(), table(), etc.,}
\DocumentationTok{\#\# omitted here {-}{-} BUT it is important that you do it, so you can catch things }
\DocumentationTok{\#\# like missing data (however it is coded), not all raters using all categories, etc.}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# Take care that all ratings run from 1 to 4, }
\DocumentationTok{\#\# whether or not rater used all categories...}
\NormalTok{tall}\SpecialCharTok{$}\NormalTok{Rating }\OtherTok{\textless{}{-}} \FunctionTok{factor}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rating,}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{)}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \FunctionTok{unique}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rubric)) \{}
\NormalTok{  ratings[,i] }\OtherTok{\textless{}{-}} \FunctionTok{factor}\NormalTok{(ratings[,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{)}
\NormalTok{\}}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# Note that in the "ratings" data frame, the missing "Sex"}
\DocumentationTok{\#\# value is "{-}{-}" while in the "tall" data frame it is ""}
\DocumentationTok{\#\# (a string of length 0).}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Make the "tall" be consistent with the "ratings" coding.}
\NormalTok{tall}\SpecialCharTok{$}\NormalTok{Sex[}\FunctionTok{nchar}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Sex)}\SpecialCharTok{==}\DecValTok{0}\NormalTok{] }\OtherTok{\textless{}{-}} \StringTok{"{-}{-}"}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# Extract the reduced data set with the 13 artifacts that all 3 raters saw...}
\NormalTok{ratings}\FloatTok{.13} \OtherTok{\textless{}{-}}\NormalTok{ ratings[}\FunctionTok{grep}\NormalTok{(}\StringTok{"O"}\NormalTok{,ratings}\SpecialCharTok{$}\NormalTok{Artifact),]}
\NormalTok{tall}\FloatTok{.13} \OtherTok{\textless{}{-}}\NormalTok{  tall[}\FunctionTok{grep}\NormalTok{(}\StringTok{"O"}\NormalTok{,tall}\SpecialCharTok{$}\NormalTok{Artifact),]}
\end{Highlighting}
\end{Shaded}

\hypertarget{a-is-the-distribution-of-ratings-for-each-rubrics-pretty-much-indistinguishable-from-the-other-rubrics-or-are-there-rubrics-that-tend-to-get-especially-high-or-low-ratings-is-the-distribution-of-ratings-given-by-each-rater-pretty-much-indistinguishable-from-the-other-raters-or-are-there-raters-that-tend-to-give-especially-high-or-low-ratings}{%
\subsection{2(a) Is the distribution of ratings for each rubrics pretty
much indistinguishable from the other rubrics, or are there rubrics that
tend to get especially high or low ratings? Is the distribution of
ratings given by each rater pretty much indistinguishable from the other
raters, or are there raters that tend to give especially high or low
ratings?}\label{a-is-the-distribution-of-ratings-for-each-rubrics-pretty-much-indistinguishable-from-the-other-rubrics-or-are-there-rubrics-that-tend-to-get-especially-high-or-low-ratings-is-the-distribution-of-ratings-given-by-each-rater-pretty-much-indistinguishable-from-the-other-raters-or-are-there-raters-that-tend-to-give-especially-high-or-low-ratings}}

Here are some ideas to compare distributions across Rubrics. I leave
other possibilities, pretty formatting (wtih kable, etc.), and smart
interpretations up to you\ldots{}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Bar plots for the reduced data set}
\NormalTok{g }\OtherTok{\textless{}{-}} \FunctionTok{ggplot}\NormalTok{(tall}\FloatTok{.13}\NormalTok{,}\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Rating)) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{( }\SpecialCharTok{\textasciitilde{}}\NormalTok{ Rubric) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{()}

\NormalTok{g}
\end{Highlighting}
\end{Shaded}

\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-7-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Table of counts might make a nice supplement}
\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\FunctionTok{lapply}\NormalTok{(}\FunctionTok{split}\NormalTok{(tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rating,tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rubric),summary))}
\FunctionTok{row.names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{paste}\NormalTok{(}\StringTok{"Rating"}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{)}

\NormalTok{tmp}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##          CritDes InitEDA InterpRes RsrchQ SelMeth TxtOrg VisOrg
## Rating 1      17       1         1      2       4      2      3
## Rating 2      16      22        18     24      29     10     22
## Rating 3       6      16        19     13       6     26     14
## Rating 4       0       0         1      0       0      1      0
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Barplots for full data set}
\NormalTok{g }\OtherTok{\textless{}{-}} \FunctionTok{ggplot}\NormalTok{(tall,}\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Rating)) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{( }\SpecialCharTok{\textasciitilde{}}\NormalTok{ Rubric) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{()}

\NormalTok{g}
\end{Highlighting}
\end{Shaded}

\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-7-2.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Table of counts again.  A bit pesky since there are NA\textquotesingle{}s...}
\NormalTok{tmp0 }\OtherTok{\textless{}{-}} \FunctionTok{lapply}\NormalTok{(}\FunctionTok{split}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rating,tall}\SpecialCharTok{$}\NormalTok{Rubric),summary)}
\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{,}\AttributeTok{nrow=}\DecValTok{5}\NormalTok{,}\AttributeTok{ncol=}\DecValTok{7}\NormalTok{))  }\DocumentationTok{\#\# seven rubrics...}
\FunctionTok{names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{names}\NormalTok{(tmp0)}
\FunctionTok{row.names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\FunctionTok{paste}\NormalTok{(}\StringTok{"Rating"}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}\StringTok{"\textless{}NA\textgreater{}"}\NormalTok{)}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \FunctionTok{names}\NormalTok{(tmp0)) \{}
\NormalTok{  tmp[,i] }\OtherTok{\textless{}{-}}\NormalTok{ tmp[,i] }\SpecialCharTok{+} \FunctionTok{c}\NormalTok{(tmp0[[i]],}\DecValTok{0}\NormalTok{)[}\DecValTok{1}\SpecialCharTok{:}\DecValTok{5}\NormalTok{]}
\NormalTok{\}}

\NormalTok{tmp}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##          CritDes InitEDA InterpRes RsrchQ SelMeth TxtOrg VisOrg
## Rating 1      47       8         6      6      10      8      7
## Rating 2      39      56        49     65      89     37     59
## Rating 3      28      47        61     45      18     66     45
## Rating 4       2       6         1      1       0      6      5
## <NA>           1       0         0      0       0      0      1
\end{verbatim}

And here are some idea to compare distributions across Raters. Again, I
leave other possibilities, attractive formatting (with kable, changing
height of graphs, etc.), and smart intepretations up to you\ldots{}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Needed to make the title of each facet more human{-}readable...}
\NormalTok{rater.name }\OtherTok{\textless{}{-}} \ControlFlowTok{function}\NormalTok{(x) \{ }\FunctionTok{paste}\NormalTok{(}\StringTok{"Rater"}\NormalTok{,x) \}}

\DocumentationTok{\#\# }
\DocumentationTok{\#\# Barplots for reduced data...}
\NormalTok{g }\OtherTok{\textless{}{-}} \FunctionTok{ggplot}\NormalTok{(tall}\FloatTok{.13}\NormalTok{,}\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Rating)) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{( }\SpecialCharTok{\textasciitilde{}}\NormalTok{ Rater, }\AttributeTok{labeller=}\FunctionTok{labeller}\NormalTok{(}\AttributeTok{Rater=}\NormalTok{rater.name)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{()}

\NormalTok{g}
\end{Highlighting}
\end{Shaded}

\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-8-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Corresponding table of counts...}
\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\FunctionTok{lapply}\NormalTok{(}\FunctionTok{split}\NormalTok{(tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rating,tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater),summary))}
\FunctionTok{row.names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{paste}\NormalTok{(}\StringTok{"Rating"}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{)}
\FunctionTok{names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{paste}\NormalTok{(}\StringTok{"Rater"}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{3}\NormalTok{)}

\NormalTok{tmp}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##          Rater 1 Rater 2 Rater 3
## Rating 1       8      10      12
## Rating 2      47      44      50
## Rating 3      35      36      29
## Rating 4       1       1       0
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Barplots for full data...}
\NormalTok{g }\OtherTok{\textless{}{-}} \FunctionTok{ggplot}\NormalTok{(tall,}\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Rating)) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{( }\SpecialCharTok{\textasciitilde{}}\NormalTok{ Rater, }\AttributeTok{labeller=}\FunctionTok{labeller}\NormalTok{(}\AttributeTok{Rater=}\NormalTok{rater.name)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{()}

\NormalTok{g}
\end{Highlighting}
\end{Shaded}

\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-8-2.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Corresponding table of counts...}
\NormalTok{tmp0 }\OtherTok{\textless{}{-}} \FunctionTok{lapply}\NormalTok{(}\FunctionTok{split}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rating,tall}\SpecialCharTok{$}\NormalTok{Rater),summary)}
\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{,}\AttributeTok{nrow=}\DecValTok{5}\NormalTok{,}\AttributeTok{ncol=}\DecValTok{3}\NormalTok{))  }\DocumentationTok{\#\# three raters...}
\FunctionTok{names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{names}\NormalTok{(tmp0)}
\FunctionTok{row.names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\FunctionTok{paste}\NormalTok{(}\StringTok{"Rating"}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}\StringTok{"\textless{}NA\textgreater{}"}\NormalTok{)}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \FunctionTok{names}\NormalTok{(tmp0)) \{}
\NormalTok{  tmp[,i] }\OtherTok{\textless{}{-}}\NormalTok{ tmp[,i] }\SpecialCharTok{+} \FunctionTok{c}\NormalTok{(tmp0[[i]],}\DecValTok{0}\NormalTok{)[}\DecValTok{1}\SpecialCharTok{:}\DecValTok{5}\NormalTok{]}
\NormalTok{\}}
\FunctionTok{names}\NormalTok{(tmp) }\OtherTok{\textless{}{-}} \FunctionTok{paste}\NormalTok{(}\StringTok{"Rater"}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{3}\NormalTok{)}
\NormalTok{tmp}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##          Rater 1 Rater 2 Rater 3
## Rating 1      29      23      40
## Rating 2     125     119     150
## Rating 3     112     120      78
## Rating 4       6      10       5
## <NA>           1       1       0
\end{verbatim}

Now, those NA's have me curious\ldots{}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{tall[}\FunctionTok{apply}\NormalTok{(tall,}\DecValTok{1}\NormalTok{,}\ControlFlowTok{function}\NormalTok{(x)\{}\FunctionTok{any}\NormalTok{(}\FunctionTok{is.na}\NormalTok{(x))\}),]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##       X Rater Artifact Repeated Semester Sex  Rubric Rating
## 161 161     2       45        0      S19   F CritDes   <NA>
## 684 684     1      100        0      F19   F  VisOrg   <NA>
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{ratings[ratings}\SpecialCharTok{$}\NormalTok{Sex}\SpecialCharTok{==}\StringTok{"{-}{-}"}\NormalTok{,]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   X Rater Sample Overlap Semester Sex RsrchQ CritDes InitEDA SelMeth InterpRes
## 5 5     3      5      NA     Fall  --      3       3       3       3         3
##   VisOrg TxtOrg Artifact Repeated
## 5      3      3        5        0
\end{verbatim}

First, note that none of the missing values occur in the smaller
13-rubric data set (how can we tell this from the output above?). So we
don't have to worry about missing data at all in analyses that just
involve this smaller data set.

Second, in any modeling that we do, the ``Rating'' is the outcome
variable, so R will just drop the two observations with missing Rating
values. This will mean that the ``full'' data sets may be different for
models that involve different rubrics: For models involving five of the
rubrics we will get all the data from all the raters, but for models
involving CritDes we would be missing a rating from Rater 2, and for
models involving VisOrg we would be missing a rating from Rater 1. We
need to be vigilant about when these differences actually occur, since
they could undermine some model comparisons (different data sets).

Third, we will also have to be careful of the missing ``Sex'' value
(currently coded as ``--''. If we coded it as NA, then R would drop it
from models that have Sex as a predictor, which would make comparing
models with and without Sex as a predictor more difficult (different
data sets!). We could just drop this student from all analyses, but it
seems like a waste to lose that data. We could code it as ``F'' or ``M''
if we had a convincing justification for doing so, but since I don't
have convincing justification (do you??), I'm just going to leave it as
a third ``Sex'' category for now\ldots{}

\hypertarget{b-for-each-rubric-do-the-raters-generally-agree-on-their-scores-if-not-is-there-one-rater-who-disagrees-with-the-others-or-do-they-all-disagree}{%
\subsection{2(b) For each rubric, do the raters generally agree on their
scores? If not, is there one rater who disagrees with the others? Or do
they all
disagree?}\label{b-for-each-rubric-do-the-raters-generally-agree-on-their-scores-if-not-is-there-one-rater-who-disagrees-with-the-others-or-do-they-all-disagree}}

Here are some coding ideas. NOTE: Once again, I leave other ideas,
attractive formatting, and interpretations (actually answering the
question!) to you\ldots{}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# useful preliminaries}
\NormalTok{Rubric.names }\OtherTok{\textless{}{-}} \FunctionTok{sort}\NormalTok{(}\FunctionTok{unique}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rubric))}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# First we examine the 13 "common" artifacts that all 3 raters saw...}

\DocumentationTok{\#\# }
\DocumentationTok{\#\# Note: extracting sig\^{}2 and tau\^{}2 from the fitted lmer() object took a little}
\DocumentationTok{\#\# spelunking...  First I fitted a "test model"}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# tmp \textless{}{-} lmer(as.numeric(Rating) \textasciitilde{} 1 + (1|Artifact),data=tall.13[tall.13$Rubric=="RsrchQ",])}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# then I looked at names(summary(tmp)) and tried to see what was under each name}
\DocumentationTok{\#\# by looking at summary(tmp)$methTitle, summary(tmp)$objClass, etc. for all the }
\DocumentationTok{\#\# names I found.  I quickly found summary(tmp)$sigma, which can be squared to get }
\DocumentationTok{\#\# sig\^{}2.  It took more exploring with summary(tmp)$varcor to get tau\^{}2...}
\DocumentationTok{\#\#}

\NormalTok{ICC.vec }\OtherTok{\textless{}{-}} \ConstantTok{NULL}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in}\NormalTok{ Rubric.names) \{}
  
\NormalTok{  tmp }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}} \DecValTok{1} \SpecialCharTok{+}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact), }\AttributeTok{data=}\NormalTok{tall}\FloatTok{.13}\NormalTok{[tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rubric}\SpecialCharTok{==}\NormalTok{i,])}
\NormalTok{  sig2 }\OtherTok{\textless{}{-}} \FunctionTok{summary}\NormalTok{(tmp)}\SpecialCharTok{$}\NormalTok{sigma}\SpecialCharTok{\^{}}\DecValTok{2}
\NormalTok{  tau2 }\OtherTok{\textless{}{-}} \FunctionTok{attr}\NormalTok{(}\FunctionTok{summary}\NormalTok{(tmp)}\SpecialCharTok{$}\NormalTok{varcor[[}\DecValTok{1}\NormalTok{]],}\StringTok{"stddev"}\NormalTok{)}\SpecialCharTok{\^{}}\DecValTok{2} 
\NormalTok{  ICC }\OtherTok{\textless{}{-}}\NormalTok{ tau2 }\SpecialCharTok{/}\NormalTok{ (tau2 }\SpecialCharTok{+}\NormalTok{ sig2)}
\NormalTok{  ICC.vec }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(ICC.vec,ICC) }
\NormalTok{\}}
\FunctionTok{names}\NormalTok{(ICC.vec) }\OtherTok{\textless{}{-}}\NormalTok{ Rubric.names}

\NormalTok{agreement.results }\OtherTok{\textless{}{-}} \FunctionTok{cbind}\NormalTok{(}\AttributeTok{ICC.common=}\NormalTok{ICC.vec,}\StringTok{"       a12"}\OtherTok{=}\DecValTok{0}\NormalTok{,}\AttributeTok{a23=}\DecValTok{0}\NormalTok{,}\AttributeTok{a13=}\DecValTok{0}\NormalTok{)}

\NormalTok{agreement.tables }\OtherTok{\textless{}{-}} \FunctionTok{as.list}\NormalTok{(}\FunctionTok{rep}\NormalTok{(}\ConstantTok{NA}\NormalTok{,}\DecValTok{7}\NormalTok{))}
\FunctionTok{names}\NormalTok{(agreement.tables) }\OtherTok{\textless{}{-}}\NormalTok{ Rubric.names}

\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in}\NormalTok{ Rubric.names) \{}
\NormalTok{  r12 }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{r1=}\FunctionTok{factor}\NormalTok{(ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{1}\NormalTok{,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}
                    \AttributeTok{r2=}\FunctionTok{factor}\NormalTok{(ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{2}\NormalTok{,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}
                    \AttributeTok{a1=}\NormalTok{ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{1}\NormalTok{,}\StringTok{"Artifact"}\NormalTok{],}
                    \AttributeTok{a2=}\NormalTok{ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{2}\NormalTok{,}\StringTok{"Artifact"}\NormalTok{])}
  \ControlFlowTok{if}\NormalTok{(}\FunctionTok{any}\NormalTok{(r12[,}\DecValTok{3}\NormalTok{]}\SpecialCharTok{!=}\NormalTok{r12[,}\DecValTok{4}\NormalTok{])) \{ }\FunctionTok{stop}\NormalTok{(}\FunctionTok{paste}\NormalTok{(}\StringTok{"Rater 1{-}2 Artifact mismatch on rubric"}\NormalTok{,i)) \}}
\NormalTok{  a12 }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(r12[,}\DecValTok{1}\NormalTok{]}\SpecialCharTok{==}\NormalTok{r12[,}\DecValTok{2}\NormalTok{])}
\NormalTok{  r12 }\OtherTok{\textless{}{-}} \FunctionTok{table}\NormalTok{(r12[,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{2}\NormalTok{])  }\DocumentationTok{\#\# print this to see how much agreement there is among raters 1{-}2}

\NormalTok{  r23 }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{r2=}\FunctionTok{factor}\NormalTok{(ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{2}\NormalTok{,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}
                    \AttributeTok{r3=}\FunctionTok{factor}\NormalTok{(ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{3}\NormalTok{,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}
                    \AttributeTok{a2=}\NormalTok{ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{2}\NormalTok{,}\StringTok{"Artifact"}\NormalTok{],}
                    \AttributeTok{a3=}\NormalTok{ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{3}\NormalTok{,}\StringTok{"Artifact"}\NormalTok{])}
  \ControlFlowTok{if}\NormalTok{(}\FunctionTok{any}\NormalTok{(r23[,}\DecValTok{3}\NormalTok{]}\SpecialCharTok{!=}\NormalTok{r23[,}\DecValTok{4}\NormalTok{])) \{ }\FunctionTok{stop}\NormalTok{(}\FunctionTok{paste}\NormalTok{(}\StringTok{"Rater 2{-}3 Artifact mismatch on rubric"}\NormalTok{,i)) \}}
\NormalTok{  a23 }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(r23[,}\DecValTok{1}\NormalTok{]}\SpecialCharTok{==}\NormalTok{r23[,}\DecValTok{2}\NormalTok{])}
\NormalTok{  r23 }\OtherTok{\textless{}{-}} \FunctionTok{table}\NormalTok{(r23[,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{2}\NormalTok{])  }\DocumentationTok{\#\# print this to see how much agreement there is among raters 2{-}3}

\NormalTok{  r13 }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{r1=}\FunctionTok{factor}\NormalTok{(ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{1}\NormalTok{,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}
                    \AttributeTok{r3=}\FunctionTok{factor}\NormalTok{(ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{3}\NormalTok{,i],}\AttributeTok{levels=}\DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{),}
                    \AttributeTok{a1=}\NormalTok{ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{1}\NormalTok{,}\StringTok{"Artifact"}\NormalTok{],}
                    \AttributeTok{a3=}\NormalTok{ratings}\FloatTok{.13}\NormalTok{[ratings}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rater}\SpecialCharTok{==}\DecValTok{3}\NormalTok{,}\StringTok{"Artifact"}\NormalTok{])}
  \ControlFlowTok{if}\NormalTok{(}\FunctionTok{any}\NormalTok{(r13[,}\DecValTok{3}\NormalTok{]}\SpecialCharTok{!=}\NormalTok{r13[,}\DecValTok{4}\NormalTok{])) \{ }\FunctionTok{stop}\NormalTok{(}\FunctionTok{paste}\NormalTok{(}\StringTok{"Rater 1{-}3 Artifact mismatch on rubric"}\NormalTok{,i)) \}}
\NormalTok{  a13 }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(r13[,}\DecValTok{1}\NormalTok{]}\SpecialCharTok{==}\NormalTok{r13[,}\DecValTok{2}\NormalTok{])}
\NormalTok{  r13 }\OtherTok{\textless{}{-}} \FunctionTok{table}\NormalTok{(r13[,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{2}\NormalTok{])  }\DocumentationTok{\#\# print this to see how much agreement there is among raters 1{-}3}
  
\NormalTok{  agreement.results[i,}\DecValTok{2}\SpecialCharTok{:}\DecValTok{4}\NormalTok{] }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(a12,a23,a13)}
  
\NormalTok{  agreement.tables[[i]] }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{(r12,r23,r13) }
  
\NormalTok{\}}
\FunctionTok{round}\NormalTok{(agreement.results,}\DecValTok{2}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##           ICC.common        a12  a23  a13
## CritDes         0.57       0.54 0.69 0.62
## InitEDA         0.49       0.69 0.85 0.54
## InterpRes       0.23       0.62 0.62 0.54
## RsrchQ          0.19       0.38 0.54 0.77
## SelMeth         0.52       0.92 0.69 0.62
## TxtOrg          0.14       0.69 0.54 0.62
## VisOrg          0.59       0.54 0.77 0.77
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}

\ControlFlowTok{if}\NormalTok{ (F) \{ }\FunctionTok{print}\NormalTok{(agreement.tables) \} }

\DocumentationTok{\#\# change to "if (T)" to get a crude printout of all tables...}

\DocumentationTok{\#\# }
\DocumentationTok{\#\# Now add in ICC\textquotesingle{}s calculated from all the data...}

\NormalTok{ICC.vec }\OtherTok{\textless{}{-}} \ConstantTok{NULL}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in}\NormalTok{ Rubric.names) \{}
  
\NormalTok{  tmp }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}} \DecValTok{1} \SpecialCharTok{+}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact), }\AttributeTok{data=}\NormalTok{tall[tall}\SpecialCharTok{$}\NormalTok{Rubric}\SpecialCharTok{==}\NormalTok{i,])}
\NormalTok{  sig2 }\OtherTok{\textless{}{-}} \FunctionTok{summary}\NormalTok{(tmp)}\SpecialCharTok{$}\NormalTok{sigma}\SpecialCharTok{\^{}}\DecValTok{2}
\NormalTok{  tau2 }\OtherTok{\textless{}{-}} \FunctionTok{attr}\NormalTok{(}\FunctionTok{summary}\NormalTok{(tmp)}\SpecialCharTok{$}\NormalTok{varcor[[}\DecValTok{1}\NormalTok{]],}\StringTok{"stddev"}\NormalTok{)}\SpecialCharTok{\^{}}\DecValTok{2} 
\NormalTok{  ICC }\OtherTok{\textless{}{-}}\NormalTok{ tau2 }\SpecialCharTok{/}\NormalTok{ (tau2 }\SpecialCharTok{+}\NormalTok{ sig2)}
\NormalTok{  ICC.vec }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(ICC.vec,ICC) }
\NormalTok{\}}
\FunctionTok{names}\NormalTok{(ICC.vec) }\OtherTok{\textless{}{-}}\NormalTok{ Rubric.names}

\NormalTok{agreement.results }\OtherTok{\textless{}{-}} \FunctionTok{cbind}\NormalTok{(}\AttributeTok{ICC.alldata=}\NormalTok{ICC.vec,agreement.results)}

\FunctionTok{round}\NormalTok{(agreement.results,}\DecValTok{2}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##           ICC.alldata ICC.common        a12  a23  a13
## CritDes          0.67       0.57       0.54 0.69 0.62
## InitEDA          0.69       0.49       0.69 0.85 0.54
## InterpRes        0.22       0.23       0.62 0.62 0.54
## RsrchQ           0.21       0.19       0.38 0.54 0.77
## SelMeth          0.47       0.52       0.92 0.69 0.62
## TxtOrg           0.19       0.14       0.69 0.54 0.62
## VisOrg           0.66       0.59       0.54 0.77 0.77
\end{verbatim}

\hypertarget{c-more-generally-how-are-the-various-factors-in-this-experiement-rater-semester-sex-repeated-rubric-related-to-the-ratings-do-the-factors-interact-in-any-interesting-ways}{%
\subsection{2(c) More generally, how are the various factors in this
experiement (Rater, Semester, Sex, Repeated, Rubric) related to the
ratings? Do the factors interact in any interesting
ways?}\label{c-more-generally-how-are-the-various-factors-in-this-experiement-rater-semester-sex-repeated-rubric-related-to-the-ratings-do-the-factors-interact-in-any-interesting-ways}}

There is a lot here, so I will break it up into ``subsections'':

\begin{itemize}
\item
  2(c)(i): Adding fixed effects to the seven rubric-specific models
  using just the data from the 13 common artifacts that al three raters
  saw
\item
  2(c)(ii): Adding fixed effects to the seven rubric-specific models
  using all the data
\item
  2(c)(iii): Trying interactions and new random effects for the seven
  rubric specific models using all the data
\item
  2(c)(iv): Trying to add fixed effects, interactions, and new random
  effects to the ``combined'' model Rating \textasciitilde{} 1 + (0 +
  Rubric\textbar Artifact), using all the data.
\end{itemize}

I want to note that some of the output below is pretty verbose (that's
why there are so many pages!), and some of it runs off the edge of the
page. I dislike both of these (verbose output, and output you can't read
because it runs off the page) but I haven't been able to fix it. Some R
functions just do not respect page layout attributes, and some functions
don't have a `turn off verbose output' option.

Also, some of the models take lmer considerable time to fit. You may
think your computer has hung. It probably has not hung; some models take
a minute or two to fit. If anything is taking more than 5 minutes,
that's the time to consider whether you really need that model.

\hypertarget{ci-adding-fixed-effects-to-the-seven-rubric-specific-models-using-just-the-data-from-the-13-common-artifacts-that-al-three-raters-saw}{%
\subsubsection{2(c)(i): Adding fixed effects to the seven
rubric-specific models using just the data from the 13 common artifacts
that al three raters
saw}\label{ci-adding-fixed-effects-to-the-seven-rubric-specific-models-using-just-the-data-from-the-13-common-artifacts-that-al-three-raters-saw}}

First, we try to add fixed effects to our seven rubric-specific
models\ldots{} In principle it will matter whether we use only the data
reduced to the 13 common artifacts, or the full data set.

I will start with the reduced data (so of course I can't check
``repeated'' on this reduced data---why not?).

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(LMERConvenienceFunctions)}
\FunctionTok{library}\NormalTok{(RLRsim)}

\DocumentationTok{\#\# Experiments before trying "production" code:}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# I started by fitting a single model and trying fitLMER.fnc() on it. }
\DocumentationTok{\#\# }
\DocumentationTok{\#\# As stated on Piazza, fitLMER.fnc() doesn\textquotesingle{}t seem to like intecept{-}only}
\DocumentationTok{\#\# as the final model, and so for models including rater, I removed }
\DocumentationTok{\#\# the intercept, so that (effectively) rater would always be in the }
\DocumentationTok{\#\# model, and fitLMER.fnc() wouldn\textquotesingle{}t complain.}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# So my starting model for experimenting was}

\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}} \SpecialCharTok{{-}}\DecValTok{1} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{              Semester }\SpecialCharTok{+}\NormalTok{ Sex }\SpecialCharTok{+}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact),}
            \AttributeTok{data=}\NormalTok{tall}\FloatTok{.13}\NormalTok{[tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rubric}\SpecialCharTok{==}\StringTok{"RsrchQ"}\NormalTok{,],}\AttributeTok{REML=}\ConstantTok{FALSE}\NormalTok{)}

\DocumentationTok{\#\#}
\DocumentationTok{\#\# Since backwards{-}elimination always involves nested models, I could}
\DocumentationTok{\#\# use t{-}tests, F{-}tests or likelihood ratio tests to eliminate fixed}
\DocumentationTok{\#\# effects.  The default for fitLMER.fnc() is to use t{-}tests with a }
\DocumentationTok{\#\# threshold of 2 (cutoff for the t{-}statistic, rather than a cutoff}
\DocumentationTok{\#\# like 0.05 for the p{-}value). This is good enough for me.  I will also }
\DocumentationTok{\#\# force fitLMER.fnc() to fit using ML rather than REML, so the}
\DocumentationTok{\#\# t{-}tests are as close to correct as I can get.}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# So a typical function call would be }

\NormalTok{tmp.back\_elim }\OtherTok{\textless{}{-}} \FunctionTok{fitLMER.fnc}\NormalTok{(tmp,}\AttributeTok{set.REML.FALSE =} \ConstantTok{TRUE}\NormalTok{,}\AttributeTok{log.file.name =} \ConstantTok{FALSE}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.7355 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.279 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# (It would be nice to eliminate the verbose narrative here, but I don\textquotesingle{}t see }
\DocumentationTok{\#\# a way to do that.  Just stuck with it I guess...)}

\DocumentationTok{\#\# Anyway, backwards elimination with fitLMER.fnc() yields a model }
\DocumentationTok{\#\# with raters only:}

\FunctionTok{formula}\NormalTok{(tmp.back\_elim)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## as.numeric(Rating) ~ as.factor(Rater) + (1 | Artifact) - 1
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# (Note that raters will always remain in the model, since raters have to rate}
\DocumentationTok{\#\# in a category \textgreater{}= 1, and the t{-}tests compares each rater\textquotesingle{}s average rating to 0. }
\DocumentationTok{\#\# Unless the sample size is really small, this should always yield a significant }
\DocumentationTok{\#\# coefficient for each rater dummy variable in the model.)}

\DocumentationTok{\#\# The estimates for raters don\textquotesingle{}t look that different from each other, }
\DocumentationTok{\#\# so we can test to see if they are different by comparing with the }
\DocumentationTok{\#\# intercept{-}only model}

\NormalTok{tmp.int\_only }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(tmp.back\_elim, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \DecValTok{1} \SpecialCharTok{{-}} \FunctionTok{as.factor}\NormalTok{(Rater))}

\FunctionTok{anova}\NormalTok{(tmp.int\_only,tmp.back\_elim)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Data: tall.13[tall.13$Rubric == "RsrchQ", ]
## Models:
## tmp.int_only: as.numeric(Rating) ~ (1 | Artifact)
## tmp.back_elim: as.numeric(Rating) ~ as.factor(Rater) + (1 | Artifact) - 1
##               npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)
## tmp.int_only     3 69.457 74.447 -31.728   63.457                     
## tmp.back_elim    5 72.018 80.335 -31.009   62.018 1.4391  2      0.487
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# Again the models are nested so I really only need to look at the p{-}value}
\DocumentationTok{\#\# from the likelihood ratio chi{-}squared test.  A little fooling around with}
\DocumentationTok{\#\# names(anova(tmp.int\_only,tmp.back\_elim)), etc., shows that I can get this }
\DocumentationTok{\#\# as }

\FunctionTok{anova}\NormalTok{(tmp.int\_only,tmp.back\_elim)}\SpecialCharTok{$}\StringTok{"Pr(\textgreater{}Chisq)"}\NormalTok{[}\DecValTok{2}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.4869707
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# it looks like the intercept{-}only model is adequate here (the p{-}value }
\DocumentationTok{\#\# is much greater than 0.05 or any other common significance level).}

\DocumentationTok{\#\# Note: since no main effects were retained, there\textquotesingle{}s really no reason to }
\DocumentationTok{\#\# check for interactions.}

\DocumentationTok{\#\# Now, I need to code this into a loop so I don\textquotesingle{}t have to do everything by hand...}


\NormalTok{Rubric.names }\OtherTok{\textless{}{-}} \FunctionTok{sort}\NormalTok{(}\FunctionTok{unique}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rubric))}

\NormalTok{model.formula}\FloatTok{.13} \OtherTok{\textless{}{-}} \FunctionTok{as.list}\NormalTok{(}\FunctionTok{rep}\NormalTok{(}\ConstantTok{NA}\NormalTok{,}\DecValTok{7}\NormalTok{))}
\FunctionTok{names}\NormalTok{(model.formula}\FloatTok{.13}\NormalTok{) }\OtherTok{\textless{}{-}}\NormalTok{ Rubric.names}

\DocumentationTok{\#\# There will be a lot of output from fitLMER.fnc() here... Sorry!}

\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in}\NormalTok{ Rubric.names) \{}
  
  \DocumentationTok{\#\# fit each base model}
\NormalTok{  rubric.data }\OtherTok{\textless{}{-}}\NormalTok{ tall}\FloatTok{.13}\NormalTok{[tall}\FloatTok{.13}\SpecialCharTok{$}\NormalTok{Rubric}\SpecialCharTok{==}\NormalTok{i,]}
\NormalTok{  tmp }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}} \SpecialCharTok{{-}}\DecValTok{1} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{              Semester }\SpecialCharTok{+}\NormalTok{ Sex }\SpecialCharTok{+}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact),}
            \AttributeTok{data=}\NormalTok{rubric.data,}\AttributeTok{REML=}\ConstantTok{FALSE}\NormalTok{)}
  
  \DocumentationTok{\#\# do backwards elimination}
\NormalTok{  tmp.back\_elim }\OtherTok{\textless{}{-}} \FunctionTok{fitLMER.fnc}\NormalTok{(tmp,}\AttributeTok{set.REML.FALSE =} \ConstantTok{TRUE}\NormalTok{,}\AttributeTok{log.file.name =} \ConstantTok{FALSE}\NormalTok{)}
  
  \DocumentationTok{\#\# check to see if the raters are significantly different from one another}
\NormalTok{  tmp.single\_intercept }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(tmp.back\_elim, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \DecValTok{1} \SpecialCharTok{{-}} \FunctionTok{as.factor}\NormalTok{(Rater))}
\NormalTok{  pval }\OtherTok{\textless{}{-}} \FunctionTok{anova}\NormalTok{(tmp.single\_intercept,tmp.back\_elim)}\SpecialCharTok{$}\StringTok{"Pr(\textgreater{}Chisq)"}\NormalTok{[}\DecValTok{2}\NormalTok{]}
  
  \DocumentationTok{\#\# choose the best model}
  \ControlFlowTok{if}\NormalTok{ (pval}\SpecialCharTok{\textless{}=}\FloatTok{0.05}\NormalTok{) \{}
\NormalTok{    tmp\_final }\OtherTok{\textless{}{-}}\NormalTok{ tmp.back\_elim}
\NormalTok{  \} }\ControlFlowTok{else}\NormalTok{ \{}
\NormalTok{    tmp\_final }\OtherTok{\textless{}{-}}\NormalTok{ tmp.single\_intercept}
\NormalTok{  \}}

  \DocumentationTok{\#\# and add to list...}
\NormalTok{  model.formula}\FloatTok{.13}\NormalTok{[[i]] }\OtherTok{\textless{}{-}} \FunctionTok{formula}\NormalTok{(tmp\_final)}
  
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.2229 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Semester" = 0.1826 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.8137 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.6429 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.8294 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.2947 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.7355 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.279 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.9383 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Semester" = 0.4287 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.5358 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.1319 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.1922 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.1078 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# see what "final models" we got...}
\NormalTok{model.formula}\FloatTok{.13}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## $CritDes
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $InitEDA
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $InterpRes
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $RsrchQ
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $SelMeth
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $TxtOrg
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $VisOrg
## as.numeric(Rating) ~ (1 | Artifact)
\end{verbatim}

So, it looks like we don't need to add any fixed effects or interactions
to the models for each rubric, using only the data reduced to the 13
common rubrics.

\hypertarget{cii-adding-fixed-effects-to-the-seven-rubric-specific-models-using-all-the-data}{%
\subsubsection{2(c)(ii): Adding fixed effects to the seven
rubric-specific models using all the
data}\label{cii-adding-fixed-effects-to-the-seven-rubric-specific-models-using-all-the-data}}

Now let's try with the full data\ldots{}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{Rubric.names }\OtherTok{\textless{}{-}} \FunctionTok{sort}\NormalTok{(}\FunctionTok{unique}\NormalTok{(tall}\SpecialCharTok{$}\NormalTok{Rubric))}

\DocumentationTok{\#\# Note: Now the missing ratings become important.  We want to use the same data}
\DocumentationTok{\#\# set for every model fit and model comparison.  I am going to eliminate by }
\DocumentationTok{\#\# hand the two observations with missing data, and only do fitting and comparison}
\DocumentationTok{\#\# on this "slightly" reduced data set.  }

\NormalTok{tall[}\FunctionTok{c}\NormalTok{(}\DecValTok{161}\NormalTok{,}\DecValTok{684}\NormalTok{),] }\DocumentationTok{\#\# just to check that these are the rows with missing ratings...}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##       X Rater Artifact Repeated Semester Sex  Rubric Rating
## 161 161     2       45        0      S19   F CritDes   <NA>
## 684 684     1      100        0      F19   F  VisOrg   <NA>
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{tall.nonmissing }\OtherTok{\textless{}{-}}\NormalTok{ tall[}\SpecialCharTok{{-}}\FunctionTok{c}\NormalTok{(}\DecValTok{161}\NormalTok{,}\DecValTok{684}\NormalTok{),]  }\DocumentationTok{\#\# now delete them...}

\DocumentationTok{\#\# I can\textquotesingle{}t think of a good justification for imputing the "Sex" of the student who}
\DocumentationTok{\#\# didn\textquotesingle{}t report this to either M or F, and leaving it as "{-}{-}" makes the models}
\DocumentationTok{\#\# harder to interpret.  So I will eliminate that person from the data set also...}

\NormalTok{tall.nonmissing[tall.nonmissing}\SpecialCharTok{$}\NormalTok{Sex}\SpecialCharTok{==}\StringTok{"{-}{-}"}\NormalTok{,]  }\DocumentationTok{\#\# check which rows will be eliminated}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##       X Rater Artifact Repeated Semester Sex    Rubric Rating
## 5     5     3        5        0      F19  --    RsrchQ      3
## 122 122     3        5        0      F19  --   CritDes      3
## 239 239     3        5        0      F19  --   InitEDA      3
## 356 356     3        5        0      F19  --   SelMeth      3
## 473 473     3        5        0      F19  -- InterpRes      3
## 590 590     3        5        0      F19  --    VisOrg      3
## 707 707     3        5        0      F19  --    TxtOrg      3
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{tall.nonmissing }\OtherTok{\textless{}{-}}\NormalTok{ tall.nonmissing[tall.nonmissing}\SpecialCharTok{$}\NormalTok{Sex}\SpecialCharTok{!=}\StringTok{"{-}{-}"}\NormalTok{,] }\DocumentationTok{\#\# eliminate them}

\NormalTok{model.formula.alldata }\OtherTok{\textless{}{-}} \FunctionTok{as.list}\NormalTok{(}\FunctionTok{rep}\NormalTok{(}\ConstantTok{NA}\NormalTok{,}\DecValTok{7}\NormalTok{))}
\FunctionTok{names}\NormalTok{(model.formula.alldata) }\OtherTok{\textless{}{-}}\NormalTok{ Rubric.names}

\DocumentationTok{\#\# There will be a lot of output from fitLMER.fnc() here... Sorry!}

\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in}\NormalTok{ Rubric.names) \{}
  
  \DocumentationTok{\#\# fit each base model}
\NormalTok{  rubric.data }\OtherTok{\textless{}{-}}\NormalTok{ tall.nonmissing[tall.nonmissing}\SpecialCharTok{$}\NormalTok{Rubric}\SpecialCharTok{==}\NormalTok{i,]}
\NormalTok{  tmp }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}} \SpecialCharTok{{-}}\DecValTok{1} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{              Semester }\SpecialCharTok{+}\NormalTok{ Sex }\SpecialCharTok{+}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact),}
            \AttributeTok{data=}\NormalTok{rubric.data,}\AttributeTok{REML=}\ConstantTok{FALSE}\NormalTok{)}
  
  \DocumentationTok{\#\# do backwards elimination}
\NormalTok{  tmp.back\_elim }\OtherTok{\textless{}{-}} \FunctionTok{fitLMER.fnc}\NormalTok{(tmp,}\AttributeTok{set.REML.FALSE =} \ConstantTok{TRUE}\NormalTok{,}\AttributeTok{log.file.name =} \ConstantTok{FALSE}\NormalTok{)}
  
  \DocumentationTok{\#\# check to see if the raters are significantly different from one another}
\NormalTok{  tmp.single\_intercept }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(tmp.back\_elim, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \DecValTok{1} \SpecialCharTok{{-}} \FunctionTok{as.factor}\NormalTok{(Rater))}
\NormalTok{  pval }\OtherTok{\textless{}{-}} \FunctionTok{anova}\NormalTok{(tmp.single\_intercept,tmp.back\_elim)}\SpecialCharTok{$}\StringTok{"Pr(\textgreater{}Chisq)"}\NormalTok{[}\DecValTok{2}\NormalTok{]}
  
  \DocumentationTok{\#\# choose the best model}
  \ControlFlowTok{if}\NormalTok{ (pval}\SpecialCharTok{\textless{}=}\FloatTok{0.05}\NormalTok{) \{}
\NormalTok{    tmp\_final }\OtherTok{\textless{}{-}}\NormalTok{ tmp.back\_elim}
\NormalTok{  \} }\ControlFlowTok{else}\NormalTok{ \{}
\NormalTok{    tmp\_final }\OtherTok{\textless{}{-}}\NormalTok{ tmp.single\_intercept}
\NormalTok{  \}}

  \DocumentationTok{\#\# and add to list...}
\NormalTok{  model.formula.alldata[[i]] }\OtherTok{\textless{}{-}} \FunctionTok{formula}\NormalTok{(tmp\_final)}
  
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.7154 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.5297 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.8802 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.7402 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.608 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Semester" = 0.5312 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.6166 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Semester" = 0.3987 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.1935 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.5041 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Semester" = 0.205 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in fitLMER.fnc(tmp, set.REML.FALSE = TRUE, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Semester" = 0.2158 >= 0.05 
##     not part of higher-order interaction
##     removing term
##   iteration 2 
##     p-value for term "Sex" = 0.3523 >= 0.05 
##     not part of higher-order interaction
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# see what "final models" we got...}
\NormalTok{model.formula.alldata}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## $CritDes
## as.numeric(Rating) ~ as.factor(Rater) + (1 | Artifact) - 1
## 
## $InitEDA
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $InterpRes
## as.numeric(Rating) ~ as.factor(Rater) + (1 | Artifact) - 1
## 
## $RsrchQ
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $SelMeth
## as.numeric(Rating) ~ as.factor(Rater) + Semester + (1 | Artifact) - 
##     1
## 
## $TxtOrg
## as.numeric(Rating) ~ (1 | Artifact)
## 
## $VisOrg
## as.numeric(Rating) ~ as.factor(Rater) + (1 | Artifact) - 1
\end{verbatim}

\hypertarget{ciii-trying-interactions-and-new-random-effects-for-the-seven-rubric-specific-models-using-all-the-data}{%
\subsubsection{2(c)(iii): Trying interactions and new random effects for
the seven rubric specific models using all the
data}\label{ciii-trying-interactions-and-new-random-effects-for-the-seven-rubric-specific-models-using-all-the-data}}

Now we see there are some differences among the models: For
\texttt{InitEDA}, \texttt{RsrchQ} and \texttt{SelMeth}, the models are
just the simple random-intercept models. For the other four, the models
are a little more complex. We should examine each of these 4 models to
see (a) if the fixed effects make sense to us; and (2) if there are any
interactions or additional random effects to consider.

I will illustrate a bit of this with the model for \texttt{SelMeth}, and
leave the rest to you.

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# refit the model and check on the t{-}statistics {-}{-} do all the variables matter?}
\NormalTok{fla }\OtherTok{\textless{}{-}} \FunctionTok{formula}\NormalTok{(model.formula.alldata[[}\StringTok{"SelMeth"}\NormalTok{]])}
\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(fla,}\AttributeTok{data=}\NormalTok{tall.nonmissing[tall.nonmissing}\SpecialCharTok{$}\NormalTok{Rubric}\SpecialCharTok{==}\StringTok{"SelMeth"}\NormalTok{,])}
\FunctionTok{round}\NormalTok{(}\FunctionTok{summary}\NormalTok{(tmp)}\SpecialCharTok{$}\NormalTok{coef,}\DecValTok{2}\NormalTok{)  }\DocumentationTok{\#\# fixed effects and their t{-}values}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##                   Estimate Std. Error t value
## as.factor(Rater)1     2.25       0.08   29.99
## as.factor(Rater)2     2.23       0.07   29.99
## as.factor(Rater)3     2.03       0.08   27.03
## SemesterS19          -0.36       0.10   -3.66
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# apparently they do.}

\DocumentationTok{\#\# now check to make sure we really need "Rater" as a factor...}
\NormalTok{tmp.single\_intercept }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(tmp, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \DecValTok{1} \SpecialCharTok{{-}} \FunctionTok{as.factor}\NormalTok{(Rater))}
\FunctionTok{anova}\NormalTok{(tmp.single\_intercept,tmp)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Data: tall.nonmissing[tall.nonmissing$Rubric == "SelMeth", ]
## Models:
## tmp.single_intercept: as.numeric(Rating) ~ Semester + (1 | Artifact)
## tmp: as.numeric(Rating) ~ as.factor(Rater) + Semester + (1 | Artifact) - 1
##                      npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)  
## tmp.single_intercept    4 145.07 156.08 -68.534   137.07                       
## tmp                     6 142.05 158.58 -65.027   130.05 7.0146  2    0.02998 *
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# looks like we do, so we keep "tmp" as our best model so far...}

\DocumentationTok{\#\# now let\textquotesingle{}s check for fixed{-}effect interactions... Since only Rater and Semester}
\DocumentationTok{\#\# are involved, we only need to examine Rater*Semester}

\NormalTok{tmp.fixed\_interactions }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(tmp, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{*}\NormalTok{Semester }\SpecialCharTok{{-}}\NormalTok{ Semester)}
\DocumentationTok{\#\# I\textquotesingle{}ve specified the model so that I can see (a) a different intercept for each}
\DocumentationTok{\#\# rater, and (b) a different semester effect for each rater.}

\FunctionTok{anova}\NormalTok{(tmp,tmp.fixed\_interactions)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Data: tall.nonmissing[tall.nonmissing$Rubric == "SelMeth", ]
## Models:
## tmp: as.numeric(Rating) ~ as.factor(Rater) + Semester + (1 | Artifact) - 1
## tmp.fixed_interactions: as.numeric(Rating) ~ as.factor(Rater) + (1 | Artifact) + as.factor(Rater):Semester - 1
##                        npar    AIC    BIC  logLik deviance Chisq Df Pr(>Chisq)
## tmp                       6 142.05 158.58 -65.027   130.05                    
## tmp.fixed_interactions    8 143.46 165.49 -63.731   127.46 2.592  2     0.2736
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# Looks like the fixed{-}effect interactions are not needed; again we keep }
\DocumentationTok{\#\# "tmp" as our best model so far... }

\DocumentationTok{\#\# Finally we check for random effects.  We should only add random effects that }
\DocumentationTok{\#\# are also present as fixed effects.  This means, for this model, we should try}
\DocumentationTok{\#\# (Rater|Artifact) and (Semester|Artifact).}

\DocumentationTok{\#\# I will show how to test these with exactRLRT()...}

\DocumentationTok{\#\# Testing (Semester|Artifact)...}

\NormalTok{m0 }\OtherTok{\textless{}{-}}\NormalTok{ tmp                                       }\DocumentationTok{\#\# Null hypothesis}
\NormalTok{mA }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(m0, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+}\NormalTok{ (Semester}\SpecialCharTok{|}\NormalTok{Artifact))   }\DocumentationTok{\#\# Alternative hypotheses}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error: number of observations (=116) <= number of random effects (=180) for term (Semester | Artifact); the random-effects parameters and the residual variance (or scale parameter) are probably unidentifiable
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m  }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(mA, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{{-}}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact))          }\DocumentationTok{\#\# Model with only the new R.E.}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error in h(simpleError(msg, call)): error in evaluating the argument 'object' in selecting a method for function 'update': object 'mA' not found
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{exactRLRT}\NormalTok{(}\AttributeTok{m0=}\NormalTok{m0,}\AttributeTok{mA=}\NormalTok{mA,}\AttributeTok{m=}\NormalTok{m)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error in exactRLRT(m0 = m0, mA = mA, m = m): object 'm' not found
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# Many error messages!  But note what the first one, for model mA is: there are}
\DocumentationTok{\#\# more random effects than there are observations in the data set!  As explained}
\DocumentationTok{\#\# on Piazza, this means lmer() cannot fit a model.  Thus, the model}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# as.numeric(Rating) \textasciitilde{} {-}1 + as.factor(Rater) + Semester + }
\DocumentationTok{\#\#                           (1 | Artifact) + (Semseter | Artifact)}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# isn\textquotesingle{}t even possible, so no testing is needed.  }



\DocumentationTok{\#\# Testng (as.factor(Rater)|Artifact)}

\NormalTok{m0 }\OtherTok{\textless{}{-}}\NormalTok{ tmp                                       }\DocumentationTok{\#\# Null hypothesis}
\NormalTok{mA }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(m0, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+}\NormalTok{ (}\FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{|}\NormalTok{Artifact))   }\DocumentationTok{\#\# Alternative hypotheses}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error: number of observations (=116) <= number of random effects (=270) for term (as.factor(Rater) | Artifact); the random-effects parameters and the residual variance (or scale parameter) are probably unidentifiable
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m  }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(mA, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{{-}}\NormalTok{ (}\DecValTok{1}\SpecialCharTok{|}\NormalTok{Artifact))          }\DocumentationTok{\#\# Model with only the new R.E.}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error in h(simpleError(msg, call)): error in evaluating the argument 'object' in selecting a method for function 'update': object 'mA' not found
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{exactRLRT}\NormalTok{(}\AttributeTok{m0=}\NormalTok{m0,}\AttributeTok{mA=}\NormalTok{mA,}\AttributeTok{m=}\NormalTok{m)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error in exactRLRT(m0 = m0, mA = mA, m = m): object 'm' not found
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# Same thing happened!  Again, the model}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# as.numeric(Rating) \textasciitilde{} {-}1 + as.factor(Rater) + Semester + }
\DocumentationTok{\#\#                           (1 | Artifact) + (as.factor(Rater) | Artifact)}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# isn\textquotesingle{}t even possible, so no testing is needed.}


\DocumentationTok{\#\# Thus, we weren\textquotesingle{}t able to add or take away anything from the model "tmp", }
\DocumentationTok{\#\# so this is our final model for SelMeth:}

\FunctionTok{summary}\NormalTok{(tmp)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linear mixed model fit by REML ['lmerMod']
## Formula: as.numeric(Rating) ~ as.factor(Rater) + Semester + (1 | Artifact) -  
##     1
##    Data: tall.nonmissing[tall.nonmissing$Rubric == "SelMeth", ]
## 
## REML criterion at convergence: 143.6
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.0480 -0.3923 -0.0551  0.2674  2.5827 
## 
## Random effects:
##  Groups   Name        Variance Std.Dev.
##  Artifact (Intercept) 0.08973  0.2996  
##  Residual             0.10842  0.3293  
## Number of obs: 116, groups:  Artifact, 90
## 
## Fixed effects:
##                   Estimate Std. Error t value
## as.factor(Rater)1  2.25037    0.07503  29.992
## as.factor(Rater)2  2.22653    0.07424  29.991
## as.factor(Rater)3  2.03316    0.07521  27.033
## SemesterS19       -0.35860    0.09796  -3.661
## 
## Correlation of Fixed Effects:
##             a.(R)1 a.(R)2 a.(R)3
## as.fctr(R)2  0.285              
## as.fctr(R)3  0.287  0.280       
## SemesterS19 -0.413 -0.391 -0.394
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# You would need to do something similar with the other models that are not }
\DocumentationTok{\#\# just intercept{-}only models (i.e the models for CritDes and InterpRes)}

\DocumentationTok{\#\# ALSO, please don\textquotesingle{}t forget:  I am not giving interpretations to the model fits}
\DocumentationTok{\#\# or coefficient estimates here. That is something I\textquotesingle{}m leaving for you, as you}
\DocumentationTok{\#\# complete your analyses and write an IDMRAD paper for the college.}
\end{Highlighting}
\end{Shaded}

\hypertarget{civ-trying-to-add-fixed-effects-interactions-and-new-random-effects-to-the-combined-model-rating-1-0-rubricartifact-using-all-the-data.}{%
\subsubsection{2(c)(iv): Trying to add fixed effects, interactions, and
new random effects to the ``combined'' model Rating \textasciitilde{} 1
+ (0 + Rubric\textbar Artifact), using all the
data.}\label{civ-trying-to-add-fixed-effects-interactions-and-new-random-effects-to-the-combined-model-rating-1-0-rubricartifact-using-all-the-data.}}

Now we try something similar with the ``combined'' model suggested on
p.~4 of the project assignment sheet.

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# Start with the "combined" intercept{-}only model...}

\NormalTok{comb}\FloatTok{.0} \OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}} \DecValTok{1} \SpecialCharTok{+}\NormalTok{ (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{|}\NormalTok{ Artifact),}
               \AttributeTok{data=}\NormalTok{tall.nonmissing)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb}\FloatTok{.0}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linear mixed model fit by REML ['lmerMod']
## Formula: as.numeric(Rating) ~ 1 + (0 + Rubric | Artifact)
##    Data: tall.nonmissing
## 
## REML criterion at convergence: 1471.7
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.0218 -0.4940 -0.0753  0.5271  3.7759 
## 
## Random effects:
##  Groups   Name            Variance Std.Dev. Corr                         
##  Artifact RubricCritDes   0.64070  0.8004                                
##           RubricInitEDA   0.38288  0.6188   0.26                         
##           RubricInterpRes 0.25658  0.5065   0.00 0.79                    
##           RubricRsrchQ    0.17398  0.4171   0.38 0.50 0.74               
##           RubricSelMeth   0.09619  0.3102   0.56 0.37 0.41 0.26          
##           RubricTxtOrg    0.40425  0.6358   0.03 0.69 0.80 0.64 0.24     
##           RubricVisOrg    0.31878  0.5646   0.17 0.78 0.76 0.60 0.29 0.79
##  Residual                 0.19477  0.4413                                
## Number of obs: 810, groups:  Artifact, 90
## 
## Fixed effects:
##             Estimate Std. Error t value
## (Intercept)  2.23210    0.04013   55.63
## optimizer (nloptwrap) convergence code: 0 (OK)
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# R complains that we have a "boundary (singular) fit", i.e. the}
\DocumentationTok{\#\# variance{-}covariance matrix for the random effects is singular }
\DocumentationTok{\#\# (not of full rank), or nearly singular.  }
\DocumentationTok{\#\#}
\DocumentationTok{\#\# What this typically means is that some of the random effects are highly}
\DocumentationTok{\#\# correlated with one another.  We can see this in the "Random effects" }
\DocumentationTok{\#\# block of summary(comb.0):  }
\DocumentationTok{\#\#}
\DocumentationTok{\#\# * The random effects for VisOrg and TxtOrg seem highly correlated with }
\DocumentationTok{\#\#   each other and with everything except for the rand. effect for SelMeth}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# * The random effects for InterpRes and InitEDA are highly correlated}
\DocumentationTok{\#\# }
\DocumentationTok{\#\# * The random effects for RsrchQ and InterpRes are highly correlated}
\DocumentationTok{\#\# }
\DocumentationTok{\#\# etc.}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# In some ways we should not be surprised: these rubrics all represent }
\DocumentationTok{\#\# features of a good research report, and we would expect that if someone}
\DocumentationTok{\#\# is good at one or two of these features, they are probably good at the }
\DocumentationTok{\#\# others.}

\DocumentationTok{\#\# Although the random effects are highly correlated, we can still proceed with }
\DocumentationTok{\#\# our variable selection...}

\DocumentationTok{\#\# Try adding fixed effects with no interactions...}

\NormalTok{comb.full }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(comb}\FloatTok{.0}\NormalTok{, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+}\NormalTok{ Semester }\SpecialCharTok{+} 
\NormalTok{                      Sex }\SpecialCharTok{+}\NormalTok{ Repeated }\SpecialCharTok{+}\NormalTok{ Rubric)}

\FunctionTok{summary}\NormalTok{(comb.full)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linear mixed model fit by REML ['lmerMod']
## Formula: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) +  
##     Semester + Sex + Repeated + Rubric
##    Data: tall.nonmissing
## 
## REML criterion at convergence: 1429.6
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.1091 -0.5065 -0.0178  0.5242  3.7932 
## 
## Random effects:
##  Groups   Name            Variance Std.Dev. Corr                         
##  Artifact RubricCritDes   0.55311  0.7437                                
##           RubricInitEDA   0.35239  0.5936   0.47                         
##           RubricInterpRes 0.17512  0.4185   0.23 0.75                    
##           RubricRsrchQ    0.16997  0.4123   0.58 0.44 0.71               
##           RubricSelMeth   0.06816  0.2611   0.39 0.60 0.74 0.41          
##           RubricTxtOrg    0.26339  0.5132   0.34 0.62 0.70 0.56 0.67     
##           RubricVisOrg    0.25809  0.5080   0.35 0.73 0.68 0.52 0.41 0.76
##  Residual                 0.18916  0.4349                                
## Number of obs: 810, groups:  Artifact, 90
## 
## Fixed effects:
##                    Estimate Std. Error t value
## (Intercept)        2.013748   0.109103  18.457
## as.factor(Rater)2  0.001977   0.054887   0.036
## as.factor(Rater)3 -0.174867   0.055045  -3.177
## SemesterS19       -0.175017   0.087850  -1.992
## SexM               0.010506   0.081271   0.129
## Repeated          -0.073586   0.098522  -0.747
## RubricInitEDA      0.547054   0.095710   5.716
## RubricInterpRes    0.587091   0.100893   5.819
## RubricRsrchQ       0.460875   0.087516   5.266
## RubricSelMeth      0.164863   0.094265   1.749
## RubricTxtOrg       0.692880   0.099523   6.962
## RubricVisOrg       0.530182   0.099136   5.348
## 
## Correlation of Fixed Effects:
##             (Intr) a.(R)2 a.(R)3 SmsS19 SexM   Repetd RbIEDA RbrcIR RbrcRQ
## as.fctr(R)2 -0.245                                                        
## as.fctr(R)3 -0.237  0.499                                                 
## SemesterS19 -0.361  0.008  0.000                                          
## SexM        -0.398 -0.026 -0.035  0.302                                   
## Repeated    -0.154  0.001 -0.003  0.079  0.009                            
## RubrcIntEDA -0.552 -0.001  0.000 -0.001  0.000  0.007                     
## RbrcIntrpRs -0.660 -0.001  0.000 -0.001  0.000 -0.009  0.734              
## RubrcRsrchQ -0.626 -0.001  0.000 -0.001  0.000 -0.039  0.585  0.756       
## RubricSlMth -0.689 -0.001  0.000 -0.001  0.000 -0.088  0.659  0.777  0.689
## RubrcTxtOrg -0.611 -0.001  0.000 -0.001  0.000  0.005  0.674  0.751  0.682
## RubricVsOrg -0.607 -0.001 -0.001 -0.002 -0.001 -0.021  0.715  0.745  0.668
##             RbrcSM RbrcTO
## as.fctr(R)2              
## as.fctr(R)3              
## SemesterS19              
## SexM                     
## Repeated                 
## RubrcIntEDA              
## RbrcIntrpRs              
## RubrcRsrchQ              
## RubricSlMth              
## RubrcTxtOrg  0.725       
## RubricVsOrg  0.680  0.750
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# It\textquotesingle{}s interesting to note that comb.full is no longer a boundary (singular) }
\DocumentationTok{\#\# fit.  Adding the fixed effects changed the residuals enough that the }
\DocumentationTok{\#\# variance{-}covariance matrix for the random effects is no longer (nearly) }
\DocumentationTok{\#\# singular.}

\NormalTok{comb.back\_elim }\OtherTok{\textless{}{-}} \FunctionTok{fitLMER.fnc}\NormalTok{(comb.full, }\AttributeTok{log.file.name =} \ConstantTok{FALSE}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in fitLMER.fnc(comb.full, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 1 
##   iteration 1 
##     p-value for term "Sex" = 0.887 >= 0.05 
##     not part of higher-order interaction
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
##     removing term
##   iteration 2 
##     p-value for term "Repeated" = 0.0919 >= 0.05 
##     not part of higher-order interaction
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
##     removing term
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.back\_elim)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linear mixed model fit by REML ['lmerMod']
## Formula: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) +  
##     Semester + Rubric
##    Data: tall.nonmissing
## 
## REML criterion at convergence: 1424.1
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -3.1200 -0.5125 -0.0173  0.5302  3.7752 
## 
## Random effects:
##  Groups   Name            Variance Std.Dev. Corr                         
##  Artifact RubricCritDes   0.55495  0.7449                                
##           RubricInitEDA   0.35064  0.5921   0.47                         
##           RubricInterpRes 0.16892  0.4110   0.23 0.75                    
##           RubricRsrchQ    0.16777  0.4096   0.59 0.44 0.70               
##           RubricSelMeth   0.06499  0.2549   0.40 0.60 0.74 0.40          
##           RubricTxtOrg    0.25615  0.5061   0.33 0.61 0.69 0.55 0.66     
##           RubricVisOrg    0.25894  0.5089   0.35 0.73 0.68 0.52 0.41 0.75
##  Residual                 0.18934  0.4351                                
## Number of obs: 810, groups:  Artifact, 90
## 
## Fixed effects:
##                     Estimate Std. Error t value
## (Intercept)        2.0084130  0.0987610  20.336
## as.factor(Rater)2  0.0003231  0.0547446   0.006
## as.factor(Rater)3 -0.1771062  0.0548892  -3.227
## SemesterS19       -0.1730357  0.0826927  -2.093
## RubricInitEDA      0.5474747  0.0957148   5.720
## RubricInterpRes    0.5864544  0.1008618   5.814
## RubricRsrchQ       0.4584082  0.0874179   5.244
## RubricSelMeth      0.1590770  0.0937771   1.696
## RubricTxtOrg       0.6930033  0.0995479   6.962
## RubricVisOrg       0.5289027  0.0990973   5.337
## 
## Correlation of Fixed Effects:
##             (Intr) a.(R)2 a.(R)3 SmsS19 RbIEDA RbrcIR RbrcRQ RbrcSM RbrcTO
## as.fctr(R)2 -0.281                                                        
## as.fctr(R)3 -0.277  0.499                                                 
## SemesterS19 -0.264  0.017  0.011                                          
## RubrcIntEDA -0.610 -0.001  0.000 -0.002                                   
## RbrcIntrpRs -0.735 -0.001  0.000  0.000  0.734                            
## RubrcRsrchQ -0.701 -0.001  0.000  0.002  0.586  0.756                     
## RubricSlMth -0.782  0.000  0.000  0.006  0.662  0.779  0.688              
## RubrcTxtOrg -0.679 -0.001  0.000 -0.001  0.674  0.751  0.682  0.728       
## RubricVsOrg -0.675 -0.001 -0.001  0.000  0.715  0.745  0.667  0.681  0.750
## optimizer (nloptwrap) convergence code: 0 (OK)
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# The final model fit is a boundary fit again, but we will proceed to try}
\DocumentationTok{\#\# interactions}

\NormalTok{comb.inter }\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(comb.back\_elim, . }\SpecialCharTok{\textasciitilde{}}\NormalTok{ . }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{*}\NormalTok{Semester}\SpecialCharTok{*}\NormalTok{Rubric)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## Model failed to converge with max|grad| = 0.00371227 (tol = 0.002, component 1)
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# This didn\textquotesingle{}t quite converge, so we will try switching optimizers and increasing}
\DocumentationTok{\#\# the number of iterations allowed...}

\NormalTok{ss }\OtherTok{\textless{}{-}} \FunctionTok{getME}\NormalTok{(comb.inter,}\FunctionTok{c}\NormalTok{(}\StringTok{"theta"}\NormalTok{,}\StringTok{"fixef"}\NormalTok{))}
\NormalTok{comb.inter.u}\OtherTok{\textless{}{-}} \FunctionTok{update}\NormalTok{(comb.inter,}\AttributeTok{start=}\NormalTok{ss,}
             \AttributeTok{control=}\FunctionTok{lmerControl}\NormalTok{(}\AttributeTok{optimizer=}\StringTok{"bobyqa"}\NormalTok{,}
                                  \AttributeTok{optCtrl=}\FunctionTok{list}\NormalTok{(}\AttributeTok{maxfun=}\FloatTok{2e5}\NormalTok{)))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# it takes a few seconds to fit, but at least we got a converged fit.}
\DocumentationTok{\#\# again, boundary fit (near{-}singular random effects variance{-}covariance mtx)}

\FunctionTok{summary}\NormalTok{(comb.inter.u)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linear mixed model fit by REML ['lmerMod']
## Formula: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) +  
##     Semester + Rubric + as.factor(Rater):Semester + as.factor(Rater):Rubric +  
##     Semester:Rubric + as.factor(Rater):Semester:Rubric
##    Data: tall.nonmissing
## Control: lmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
## 
## REML criterion at convergence: 1424.4
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.9141 -0.5141 -0.0653  0.5023  3.6609 
## 
## Random effects:
##  Groups   Name            Variance Std.Dev. Corr                         
##  Artifact RubricCritDes   0.48550  0.6968                                
##           RubricInitEDA   0.35257  0.5938   0.42                         
##           RubricInterpRes 0.14619  0.3824   0.32 0.80                    
##           RubricRsrchQ    0.16444  0.4055   0.66 0.43 0.72               
##           RubricSelMeth   0.06297  0.2509   0.45 0.64 0.78 0.49          
##           RubricTxtOrg    0.25441  0.5044   0.44 0.65 0.67 0.60 0.62     
##           RubricVisOrg    0.25527  0.5052   0.35 0.73 0.68 0.57 0.35 0.76
##  Residual                 0.18839  0.4340                                
## Number of obs: 810, groups:  Artifact, 90
## 
## Fixed effects:
##                                                Estimate Std. Error t value
## (Intercept)                                    1.739538   0.136568  12.738
## as.factor(Rater)2                              0.302995   0.155107   1.953
## as.factor(Rater)3                              0.237851   0.155863   1.526
## SemesterS19                                   -0.129077   0.250318  -0.516
## RubricInitEDA                                  0.765215   0.165241   4.631
## RubricInterpRes                                0.979228   0.162160   6.039
## RubricRsrchQ                                   0.710427   0.147386   4.820
## RubricSelMeth                                  0.462750   0.155274   2.980
## RubricTxtOrg                                   1.011251   0.160899   6.285
## RubricVisOrg                                   0.647869   0.166603   3.889
## as.factor(Rater)2:SemesterS19                  0.268014   0.303883   0.882
## as.factor(Rater)3:SemesterS19                 -0.072789   0.301026  -0.242
## as.factor(Rater)2:RubricInitEDA               -0.325018   0.204108  -1.592
## as.factor(Rater)3:RubricInitEDA               -0.374190   0.205354  -1.822
## as.factor(Rater)2:RubricInterpRes             -0.469281   0.201051  -2.334
## as.factor(Rater)3:RubricInterpRes             -0.711515   0.202316  -3.517
## as.factor(Rater)2:RubricRsrchQ                -0.447050   0.189326  -2.361
## as.factor(Rater)3:RubricRsrchQ                -0.474411   0.190681  -2.488
## as.factor(Rater)2:RubricSelMeth               -0.301450   0.193678  -1.556
## as.factor(Rater)3:RubricSelMeth               -0.365656   0.194970  -1.875
## as.factor(Rater)2:RubricTxtOrg                -0.449164   0.200927  -2.235
## as.factor(Rater)3:RubricTxtOrg                -0.407754   0.202209  -2.016
## as.factor(Rater)2:RubricVisOrg                 0.009042   0.205059   0.044
## as.factor(Rater)3:RubricVisOrg                -0.287443   0.206299  -1.393
## SemesterS19:RubricInitEDA                     -0.050212   0.301475  -0.167
## SemesterS19:RubricInterpRes                    0.127813   0.295706   0.432
## SemesterS19:RubricRsrchQ                       0.133874   0.267750   0.500
## SemesterS19:RubricSelMeth                     -0.089616   0.282837  -0.317
## SemesterS19:RubricTxtOrg                       0.166097   0.293176   0.567
## SemesterS19:RubricVisOrg                       0.146845   0.302496   0.485
## as.factor(Rater)2:SemesterS19:RubricInitEDA    0.020326   0.392376   0.052
## as.factor(Rater)3:SemesterS19:RubricInitEDA    0.252422   0.389961   0.647
## as.factor(Rater)2:SemesterS19:RubricInterpRes -0.266618   0.385390  -0.692
## as.factor(Rater)3:SemesterS19:RubricInterpRes -0.152392   0.383354  -0.398
## as.factor(Rater)2:SemesterS19:RubricRsrchQ    -0.217348   0.360414  -0.603
## as.factor(Rater)3:SemesterS19:RubricRsrchQ     0.354319   0.357388   0.991
## as.factor(Rater)2:SemesterS19:RubricSelMeth   -0.401036   0.370200  -1.083
## as.factor(Rater)3:SemesterS19:RubricSelMeth   -0.192670   0.367887  -0.524
## as.factor(Rater)2:SemesterS19:RubricTxtOrg    -0.542267   0.385011  -1.408
## as.factor(Rater)3:SemesterS19:RubricTxtOrg    -0.316395   0.382614  -0.827
## as.factor(Rater)2:SemesterS19:RubricVisOrg    -0.603626   0.392909  -1.536
## as.factor(Rater)3:SemesterS19:RubricVisOrg    -0.186749   0.390759  -0.478
\end{verbatim}

\begin{verbatim}
## 
## Correlation matrix not shown by default, as p = 42 > 12.
## Use print(x, correlation=TRUE)  or
##     vcov(x)        if you need it
\end{verbatim}

\begin{verbatim}
## optimizer (bobyqa) convergence code: 0 (OK)
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# If you compare with summary(comb.inter) you will see that}
\DocumentationTok{\#\# there wasn\textquotesingle{}t much difference in the fitted values; we could }
\DocumentationTok{\#\# probably have just proceeded wth the model comb.inter.  But }
\DocumentationTok{\#\# since we have the converged model we will use it for fixed  }
\DocumentationTok{\#\# effects selection}

\NormalTok{comb.inter\_elim }\OtherTok{\textless{}{-}} \FunctionTok{fitLMER.fnc}\NormalTok{(comb.inter.u, }\AttributeTok{log.file.name =} \ConstantTok{FALSE}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in fitLMER.fnc(comb.inter.u, log.file.name = FALSE): Argument "ran.effects" is empty, which means you will not be forward-fitting the random effect structure of your model. You could just as well run function "bfFixefLMER_F.fnc" or "bfFixefLMER_t.fnc".
## TRUE
\end{verbatim}

\begin{verbatim}
## ======================================================
## ===              backfitting fixed effects         ===
## ======================================================
## processing model terms of interaction level 3 
##   iteration 1 
##     p-value for term "as.factor(Rater):Semester:Rubric" = 0.5526 >= 0.05 
##     not part of higher-order interaction
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
##     removing term
## processing model terms of interaction level 2 
##   iteration 2 
##     p-value for term "as.factor(Rater):Semester" = 0.598 >= 0.05 
##     not part of higher-order interaction
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
##     removing term
##   iteration 3 
##     p-value for term "Semester:Rubric" = 0.0761 >= 0.05 
##     not part of higher-order interaction
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
##     removing term
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## pruning random effects structure ...
##   nothing to prune
## ======================================================
## ===            forwardfitting random effects       ===
## ======================================================
##  ===       random slopes       ===
## ======================================================
## ===            re-backfitting fixed effects        ===
## ======================================================
## processing model terms of interaction level 2 
##   all terms of interaction level 2 significant
## processing model terms of interaction level 1 
##   all terms of interaction level 1 significant
## resetting REML to TRUE
\end{verbatim}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{verbatim}
## pruning random effects structure ...
##   nothing to prune
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.inter\_elim)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linear mixed model fit by REML ['lmerMod']
## Formula: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) +  
##     Semester + Rubric + as.factor(Rater):Rubric
##    Data: tall.nonmissing
## Control: lmerControl(optimizer = "bobyqa", optCtrl = list(maxfun = 2e+05))
## 
## REML criterion at convergence: 1419.6
## 
## Scaled residuals: 
##     Min      1Q  Median      3Q     Max 
## -2.9280 -0.5122 -0.0447  0.4827  3.5854 
## 
## Random effects:
##  Groups   Name            Variance Std.Dev. Corr                         
##  Artifact RubricCritDes   0.50348  0.7096                                
##           RubricInitEDA   0.35480  0.5956   0.44                         
##           RubricInterpRes 0.15192  0.3898   0.35 0.82                    
##           RubricRsrchQ    0.17953  0.4237   0.63 0.44 0.72               
##           RubricSelMeth   0.06727  0.2594   0.42 0.60 0.74 0.36          
##           RubricTxtOrg    0.26069  0.5106   0.42 0.64 0.67 0.55 0.64     
##           RubricVisOrg    0.25491  0.5049   0.34 0.71 0.68 0.51 0.38 0.77
##  Residual                 0.18519  0.4303                                
## Number of obs: 810, groups:  Artifact, 90
## 
## Fixed effects:
##                                   Estimate Std. Error t value
## (Intercept)                        1.75945    0.11785  14.929
## as.factor(Rater)2                  0.36537    0.13296   2.748
## as.factor(Rater)3                  0.21421    0.13297   1.611
## SemesterS19                       -0.17780    0.08228  -2.161
## RubricInitEDA                      0.74625    0.13676   5.457
## RubricInterpRes                    1.01453    0.13479   7.527
## RubricRsrchQ                       0.74926    0.12419   6.033
## RubricSelMeth                      0.42672    0.13040   3.272
## RubricTxtOrg                       1.04967    0.13551   7.746
## RubricVisOrg                       0.68354    0.13947   4.901
## as.factor(Rater)2:RubricInitEDA   -0.30843    0.17249  -1.788
## as.factor(Rater)3:RubricInitEDA   -0.29522    0.17282  -1.708
## as.factor(Rater)2:RubricInterpRes -0.53674    0.17008  -3.156
## as.factor(Rater)3:RubricInterpRes -0.75247    0.17049  -4.414
## as.factor(Rater)2:RubricRsrchQ    -0.50157    0.16151  -3.106
## as.factor(Rater)3:RubricRsrchQ    -0.37068    0.16179  -2.291
## as.factor(Rater)2:RubricSelMeth   -0.39602    0.16467  -2.405
## as.factor(Rater)3:RubricSelMeth   -0.41324    0.16504  -2.504
## as.factor(Rater)2:RubricTxtOrg    -0.58380    0.17141  -3.406
## as.factor(Rater)3:RubricTxtOrg    -0.48649    0.17177  -2.832
## as.factor(Rater)2:RubricVisOrg    -0.14444    0.17442  -0.828
## as.factor(Rater)3:RubricVisOrg    -0.33380    0.17481  -1.910
\end{verbatim}

\begin{verbatim}
## 
## Correlation matrix not shown by default, as p = 22 > 12.
## Use print(x, correlation=TRUE)  or
##     vcov(x)        if you need it
\end{verbatim}

\begin{verbatim}
## optimizer (bobyqa) convergence code: 0 (OK)
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# it\textquotesingle{}s a little hard to compare summaries for such big models, so let\textquotesingle{}s look }
\DocumentationTok{\#\# at the highlights:}

\FunctionTok{formula}\NormalTok{(comb.inter.u)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + 
##     Semester + Rubric + as.factor(Rater):Semester + as.factor(Rater):Rubric + 
##     Semester:Rubric + as.factor(Rater):Semester:Rubric
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{formula}\NormalTok{(comb.inter\_elim)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + 
##     Semester + Rubric + as.factor(Rater):Rubric
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{formula}\NormalTok{(comb.back\_elim)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + 
##     Semester + Rubric
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.inter.u)}\SpecialCharTok{$}\NormalTok{varcor}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  Groups   Name            Std.Dev. Corr                               
##  Artifact RubricCritDes   0.69678                                     
##           RubricInitEDA   0.59378  0.416                              
##           RubricInterpRes 0.38235  0.324 0.800                        
##           RubricRsrchQ    0.40551  0.655 0.430 0.723                  
##           RubricSelMeth   0.25094  0.446 0.639 0.784 0.488            
##           RubricTxtOrg    0.50439  0.436 0.649 0.667 0.604 0.622      
##           RubricVisOrg    0.50524  0.349 0.727 0.675 0.567 0.346 0.757
##  Residual                 0.43404
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.inter\_elim)}\SpecialCharTok{$}\NormalTok{varcor}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  Groups   Name            Std.Dev. Corr                               
##  Artifact RubricCritDes   0.70956                                     
##           RubricInitEDA   0.59565  0.445                              
##           RubricInterpRes 0.38977  0.354 0.815                        
##           RubricRsrchQ    0.42371  0.631 0.440 0.716                  
##           RubricSelMeth   0.25937  0.424 0.601 0.737 0.364            
##           RubricTxtOrg    0.51058  0.417 0.637 0.675 0.547 0.636      
##           RubricVisOrg    0.50489  0.339 0.715 0.677 0.512 0.376 0.772
##  Residual                 0.43034
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.back\_elim)}\SpecialCharTok{$}\NormalTok{varcor}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  Groups   Name            Std.Dev. Corr                               
##  Artifact RubricCritDes   0.74495                                     
##           RubricInitEDA   0.59215  0.467                              
##           RubricInterpRes 0.41100  0.230 0.749                        
##           RubricRsrchQ    0.40960  0.588 0.436 0.704                  
##           RubricSelMeth   0.25493  0.399 0.603 0.736 0.397            
##           RubricTxtOrg    0.50612  0.335 0.614 0.691 0.551 0.656      
##           RubricVisOrg    0.50886  0.350 0.731 0.679 0.516 0.414 0.752
##  Residual                 0.43513
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{anova}\NormalTok{(comb.back\_elim,comb.inter\_elim,comb.inter.u)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Data: tall.nonmissing
## Models:
## comb.back_elim: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + Semester + Rubric
## comb.inter_elim: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Rubric
## comb.inter.u: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Semester + as.factor(Rater):Rubric + Semester:Rubric + as.factor(Rater):Semester:Rubric
##                 npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)    
## comb.back_elim    39 1464.0 1647.2 -693.02   1386.0                         
## comb.inter_elim   51 1454.5 1694.1 -676.26   1352.5 33.526 12   0.000801 ***
## comb.inter.u      71 1471.4 1804.8 -664.68   1329.4 23.161 20   0.280962    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# the models are nested so we can use AIC, BIC or likelihod ratio (deviance) }
\DocumentationTok{\#\# tests... AIC and the LRT agree on comb.inter\_elim; BIC likes the simpler}
\DocumentationTok{\#\# comb.back\_elim.}

\DocumentationTok{\#\# Interestingly, comb.inter\_elim adds a rater x rubric interaction to }
\DocumentationTok{\#\# the main{-}effects model comb.back\_elim.  This suggests that the raters }
\DocumentationTok{\#\# do not all use the rubrics in the same way.}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# In addition to looking at the fixed effect coefficients in}
\DocumentationTok{\#\# summary(comb.inter\_elim)$coef, we could also see if there\textquotesingle{}s}
\DocumentationTok{\#\# a pattern in an appropriate facets plot}

\NormalTok{g }\OtherTok{\textless{}{-}} \FunctionTok{ggplot}\NormalTok{(tall.nonmissing, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x=}\NormalTok{Rating)) }\SpecialCharTok{+} 
  \FunctionTok{geom\_bar}\NormalTok{() }\SpecialCharTok{+} 
  \FunctionTok{facet\_wrap}\NormalTok{( }\SpecialCharTok{\textasciitilde{}}\NormalTok{ Rubric }\SpecialCharTok{+}\NormalTok{ Rater, }\AttributeTok{nrow=}\DecValTok{7}\NormalTok{)}

\NormalTok{g}
\end{Highlighting}
\end{Shaded}

\includegraphics{hw10_solutions_files/figure-latex/unnamed-chunk-15-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# and it does look as if the 3 raters have different ways of scoring the 7 rubrics,}
\DocumentationTok{\#\# so the interaction we found in comb.inter\_elim makes sense.  (Clearly it is}
\DocumentationTok{\#\# not the case that one rater is simply more harsh than another, or something }
\DocumentationTok{\#\# like that.  Can you describe the different patterns of scoring among the 3 }
\DocumentationTok{\#\# raters?  Can you relate it to coefficient estimates in summary()$coef?)}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# Finally, we consider adding random effects to what seems like the }
\DocumentationTok{\#\# best model so far, comb.inter\_elim...}

\DocumentationTok{\#\# The fixed{-}effects terms we have to work with are:}
\DocumentationTok{\#\# }
\DocumentationTok{\#\# as.factor(Rater) }
\DocumentationTok{\#\# Semester}
\DocumentationTok{\#\# as.factor(Rater):Rubric}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# We want to add each of these *without* a random intercept, to preserve the }
\DocumentationTok{\#\# structure of the model (separate random interepts for each rubric)}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# In all cases, there is more than one random effect to test (3 for raters, }
\DocumentationTok{\#\# 2 for semesters, 7 for rubrics, and 21 for the interaction).  Since exactRLRT()}
\DocumentationTok{\#\# can only test single random effects, we can\textquotesingle{}t use it.  Instead we inspect AIC }
\DocumentationTok{\#\# andBIC from anova() tables for these...}

\DocumentationTok{\#\# Fitting some of these models produces various errors and warnings; I am not}
\DocumentationTok{\#\# going to worry about them too much, in order to get an idea of what random}
\DocumentationTok{\#\# effects I may want...}

\NormalTok{m0 }\OtherTok{\textless{}{-}}\NormalTok{ comb.inter\_elim }
\NormalTok{mA }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}}\NormalTok{ (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} 
\NormalTok{             (}\DecValTok{0} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{             Semester }\SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{:}\NormalTok{Rubric, }\AttributeTok{data=}\NormalTok{tall.nonmissing)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{anova}\NormalTok{(m0,mA)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Warning in commonArgs(par, fn, control, environment()): maxfun < 10 *
## length(par)^2 is not recommended.
\end{verbatim}

\begin{verbatim}
## Data: tall.nonmissing
## Models:
## m0: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Rubric
## mA: as.numeric(Rating) ~ (0 + Rubric | Artifact) + (0 + as.factor(Rater) | Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Rubric
##    npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)    
## m0   51 1454.5 1694.1 -676.26   1352.5                         
## mA   57 1415.9 1683.6 -650.94   1301.9 50.647  6  3.487e-09 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# }
\DocumentationTok{\#\# AIC and BIC both like including (0 + as.factor(Rater) | Artifact) in the model}

\NormalTok{m0 }\OtherTok{\textless{}{-}}\NormalTok{ comb.inter\_elim }
\NormalTok{mA }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}}\NormalTok{ (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} 
\NormalTok{             (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ Semester }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{             Semester }\SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{:}\NormalTok{Rubric, }\AttributeTok{data=}\NormalTok{tall.nonmissing)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## unable to evaluate scaled gradient
\end{verbatim}

\begin{verbatim}
## Warning in checkConv(attr(opt, "derivs"), opt$par, ctrl = control$checkConv, :
## Model failed to converge: degenerate Hessian with 1 negative eigenvalues
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{anova}\NormalTok{(m0,mA)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## refitting model(s) with ML (instead of REML)
\end{verbatim}

\begin{verbatim}
## Data: tall.nonmissing
## Models:
## m0: as.numeric(Rating) ~ (0 + Rubric | Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Rubric
## mA: as.numeric(Rating) ~ (0 + Rubric | Artifact) + (0 + Semester | Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Rubric
##    npar    AIC    BIC  logLik deviance  Chisq Df Pr(>Chisq)
## m0   51 1454.5 1694.1 -676.26   1352.5                     
## mA   54 1458.4 1712.0 -675.18   1350.4 2.1534  3     0.5412
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#}
\DocumentationTok{\#\# AIC and BIC do not like (0 + Semester | Artifact) in the model...}


\NormalTok{m0 }\OtherTok{\textless{}{-}}\NormalTok{ comb.inter\_elim }
\NormalTok{mA }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}}\NormalTok{ (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} 
\NormalTok{             (}\DecValTok{0} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} 
\NormalTok{             (}\DecValTok{0} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{:}\NormalTok{Rubric }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{             Semester }\SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{:}\NormalTok{Rubric, }\AttributeTok{data=}\NormalTok{tall.nonmissing)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error: number of observations (=810) <= number of random effects (=1890) for term (0 + as.factor(Rater):Rubric | Artifact); the random-effects parameters and the residual variance (or scale parameter) are probably unidentifiable
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# anova(m0,mA)     {-}{-} Not needed!}
\DocumentationTok{\#\# }
\DocumentationTok{\#\# There are not enough observations to fit mA here, so we need not do any }
\DocumentationTok{\#\# formal model comparison...}

\DocumentationTok{\#\# So, to summarize, the "final" model appears to be}

\NormalTok{comb.final }\OtherTok{\textless{}{-}} \FunctionTok{lmer}\NormalTok{(}\FunctionTok{as.numeric}\NormalTok{(Rating) }\SpecialCharTok{\textasciitilde{}}\NormalTok{ (}\DecValTok{0} \SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} 
\NormalTok{             (}\DecValTok{0} \SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{|}\NormalTok{ Artifact) }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater) }\SpecialCharTok{+} 
\NormalTok{             Semester }\SpecialCharTok{+}\NormalTok{ Rubric }\SpecialCharTok{+} \FunctionTok{as.factor}\NormalTok{(Rater)}\SpecialCharTok{:}\NormalTok{Rubric, }\AttributeTok{data=}\NormalTok{tall.nonmissing)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## boundary (singular) fit: see ?isSingular
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{formula}\NormalTok{(comb.final)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## as.numeric(Rating) ~ (0 + Rubric | Artifact) + (0 + as.factor(Rater) | 
##     Artifact) + as.factor(Rater) + Semester + Rubric + as.factor(Rater):Rubric
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.final)}\SpecialCharTok{$}\NormalTok{varcor}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  Groups     Name              Std.Dev. Corr                              
##  Artifact   RubricCritDes     0.70461                                    
##             RubricInitEDA     0.56379   0.318                            
##             RubricInterpRes   0.31946   0.142  0.674                     
##             RubricRsrchQ      0.42310   0.500  0.194  0.538              
##             RubricSelMeth     0.19557   0.145  0.226  0.376 -0.240       
##             RubricTxtOrg      0.50026   0.268  0.437  0.364  0.305  0.213
##             RubricVisOrg      0.48201   0.175  0.504  0.445  0.276 -0.161
##  Artifact.1 as.factor(Rater)1 0.11320                                    
##             as.factor(Rater)2 0.33427  -0.486                            
##             as.factor(Rater)3 0.30681   0.332  0.663                     
##  Residual                     0.36699                                    
##        
##        
##        
##        
##        
##        
##        
##   0.537
##        
##        
##        
## 
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(comb.final)}\SpecialCharTok{$}\NormalTok{coef}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##                                     Estimate Std. Error    t value
## (Intercept)                        1.7575545 0.11404151 15.4115336
## as.factor(Rater)2                  0.3660542 0.13918252  2.6300297
## as.factor(Rater)3                  0.1959088 0.12966636  1.5108686
## SemesterS19                       -0.1591805 0.07647529 -2.0814634
## RubricInitEDA                      0.7394940 0.12996076  5.6901329
## RubricInterpRes                    0.9915148 0.12770767  7.7639406
## RubricRsrchQ                       0.7261869 0.11793023  6.1577676
## RubricSelMeth                      0.4106797 0.12470498  3.2932102
## RubricTxtOrg                       1.0157815 0.12999540  7.8139797
## RubricVisOrg                       0.6542506 0.13353098  4.8996162
## as.factor(Rater)2:RubricInitEDA   -0.2998076 0.15609075 -1.9207264
## as.factor(Rater)3:RubricInitEDA   -0.2947319 0.15635201 -1.8850532
## as.factor(Rater)2:RubricInterpRes -0.5132297 0.15348482 -3.3438467
## as.factor(Rater)3:RubricInterpRes -0.7148433 0.15363960 -4.6527283
## as.factor(Rater)2:RubricRsrchQ    -0.4874137 0.14722146 -3.3107521
## as.factor(Rater)3:RubricRsrchQ    -0.3223799 0.14726517 -2.1891116
## as.factor(Rater)2:RubricSelMeth   -0.3863739 0.15030941 -2.5705236
## as.factor(Rater)3:RubricSelMeth   -0.3871581 0.14961457 -2.5877033
## as.factor(Rater)2:RubricTxtOrg    -0.5510439 0.15646043 -3.5219379
## as.factor(Rater)3:RubricTxtOrg    -0.4448937 0.15673122 -2.8385772
## as.factor(Rater)2:RubricVisOrg    -0.1048994 0.15861081 -0.6613632
## as.factor(Rater)3:RubricVisOrg    -0.2752130 0.15884865 -1.7325485
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# if we accept comb.final as our final model, we can interpret the pieces as }
\DocumentationTok{\#\# follows:}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# (0 + as.factor(Rater) | Artifact) + as.factor(Rater)}
\DocumentationTok{\#\#   * There is a kind of Rater x Artifact interaction: each Rater\textquotesingle{}s}
\DocumentationTok{\#\#     rating on each Artifact differs from what we would expect (from the }
\DocumentationTok{\#\#     fixed effects alone) by a small random effect that depends on the Artifact  }
\DocumentationTok{\#\#}
\DocumentationTok{\#\# Rubric + as.factor(Rater) + as.factor(Rater):Rubric}
\DocumentationTok{\#\#   * There is a Rater x Rubric interaction: each Rater uses each}
\DocumentationTok{\#\#     Rubric in a way that is not like, or even parallel to, other rater\textquotesingle{}s }
\DocumentationTok{\#\#     Rubric usage.  (we saw that in the facets plot above also).}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# (0 + Rubric | Artifact) + Rubric}
\DocumentationTok{\#\#   * There is a kind of Rubric x Artifact interaction: There are }
\DocumentationTok{\#\#     different average scores on each rubric, but the rubric averages also}
\DocumentationTok{\#\#     vary a bit from one Artifact to the next, by a small random effect that}
\DocumentationTok{\#\#     depends on Artifact}

\DocumentationTok{\#\# In all of this, the fact that Rubric scores depend on Artifact (that is, }
\DocumentationTok{\#\# there is a kind of Rubric x Artifact interaction) is what we might expect: }
\DocumentationTok{\#\# the artifacts aren\textquotesingle{}t all of equal quality on each rubric, and so we should }
\DocumentationTok{\#\# expect the average scores on each Rubric to vary from one Artifact to the next.}
\DocumentationTok{\#\#}
\DocumentationTok{\#\# More troubling are the Rater x Rubric interaction and the "kind of" }
\DocumentationTok{\#\# Rater x Artifact interaction.  The Rater x Rubric interaction suggests }
\DocumentationTok{\#\# that the Raters are not all interpreting the Rubrics in the same way.  The }
\DocumentationTok{\#\# "kind of" Rater x Artifact interaction suggests that the Raters are not }
\DocumentationTok{\#\# interpreting the evidence in the artifacts in the same way.  These }
\DocumentationTok{\#\# interactions suggest that perhaps the raters should be trained more, to}
\DocumentationTok{\#\# make the raters\textquotesingle{} ratings more similar to each other.}
\end{Highlighting}
\end{Shaded}

\hypertarget{d-is-there-anything-else-interesting-to-say-about-this-data}{%
\subsection{2(d) Is there anything else interesting to say about this
data?}\label{d-is-there-anything-else-interesting-to-say-about-this-data}}

I leave this entirely up to you. By now you are pretty familiar with the
data (probably sick of it actually!), so you may already have some ideas
here.

Here are some things you could try (you don't need to try all of them,
and there may be other, better things to try instead. Just try to come
up with something that is interesting and useful to the college):

\begin{itemize}
\item
  Do some additional EDA. We really haven't tried to do a very complete
  EDA on this data yet.
\item
  Are there better ways to illustrate or interpret the models we have
  fitted?
\item
  Are there ways (through EDA, plots of model fits, etc.) to understand
  the differences in the models fitted to the data from the 13 common
  items, vs fitting to all the data?
\item
  Is there any way to justify, based on the data and on where the data
  come from, imputing the missing Sex value as ``M'' or ``F''?
\item
  For some of the models, residual diagnostic plots might be
  interesting.
\item
  What other models could you fit? More multi-level models? Some
  ordinary regression models? glm's? etc\ldots{}
\item
  etc.
\end{itemize}

I look forward to anything interesting that you can come up with!

\end{document}
