# Inter-rater-reliability and Intra-class-correlation

## Install required packages

DescTools

wants <- c("DescTools")
has   <- wants %in% rownames(installed.packages())
if(any(!has)) install.packages(wants[!has])

## Inter-rater-reliability

### Absolute agreement between two or more raters

#### Two raters

categ <- c("V", "N", "P")
lvls  <- factor(categ, levels=categ)
rtr1  <- rep(lvls, c(60, 30, 10))
rtr2  <- rep(rep(lvls, nlevels(lvls)), c(53,5,2, 11,14,5, 1,6,3))
cTab  <- table(rtr1, rtr2)
addmargins(cTab)
     rtr2
rtr1    V   N   P Sum
V    53   5   2  60
N    11  14   5  30
P     1   6   3  10
Sum  65  25  10 100
library(DescTools)
Agree(cbind(rtr1, rtr2))
[1] 0.7
attr(,"subjects")
[1] 100
attr(,"raters")
[1] 2

#### Three raters

rtr3 <- rep(rep(lvls, nlevels(lvls)), c(48,8,3, 15,10,7, 3,4,2))
Agree(cbind(rtr1, rtr2, rtr3))
[1] 0.6
attr(,"subjects")
[1] 100
attr(,"raters")
[1] 3

### Cohen’s unweighted $$\kappa$$ for two raters

cTab <- xtabs(~ rtr1 + rtr2)
library(DescTools)
CohenKappa(cTab, conf.conf.level=0.95)
[1] 0.4285714

### Cohen’s weighted $$\kappa$$ for two raters

Ordered categories

categ <- c("<10%", "11-20%", "21-30%", "31-40%", "41-50%", ">50%")
lvls  <- factor(categ, levels=categ)
tv1   <- rep(lvls, c(22, 21, 23, 16, 10, 8))
tv2   <- rep(rep(lvls, nlevels(lvls)), c(5,8,1,2,4,2, 3,5,3,5,5,0, 1,2,6,11,2,1,
0,1,5,4,3,3, 0,0,1,2,5,2, 0,0,1, 2,1,4))
cTab  <- table(tv1, tv2)
addmargins(cTab)
        tv2
tv1      <10% 11-20% 21-30% 31-40% 41-50% >50% Sum
<10%      5      8      1      2      4    2  22
11-20%    3      5      3      5      5    0  21
21-30%    1      2      6     11      2    1  23
31-40%    0      1      5      4      3    3  16
41-50%    0      0      1      2      5    2  10
>50%      0      0      1      2      1    4   8
Sum       9     16     17     26     20   12 100
cTab <- xtabs(~ tv1 + tv2)
library(DescTools)
CohenKappa(cTab, weights="Equal-Spacing", conf.level=0.95)
     kappa     lwr.ci     upr.ci
0.31566846 0.07905284 0.55228407 

### Fleiss’ $$\kappa$$ for two or more raters

rtr1 <- letters[c(4,2,2,5,2, 1,3,1,1,5, 1,1,2,1,2, 3,1,1,2,1, 5,2,2,1,1, 2,1,2,1,5)]
rtr2 <- letters[c(4,2,3,5,2, 1,3,1,1,5, 4,2,2,4,2, 3,1,1,2,3, 5,4,2,1,4, 2,1,2,3,5)]
rtr3 <- letters[c(4,2,3,5,2, 3,3,3,4,5, 4,4,2,4,4, 3,1,1,4,3, 5,4,4,4,4, 2,1,4,3,5)]
rtr4 <- letters[c(4,5,3,5,4, 3,3,3,4,5, 4,4,3,4,4, 3,4,1,4,5, 5,4,5,4,4, 2,1,4,3,5)]
rtr5 <- letters[c(4,5,3,5,4, 3,5,3,4,5, 4,4,3,4,4, 3,5,1,4,5, 5,4,5,4,4, 2,5,4,3,5)]
rtr6 <- letters[c(4,5,5,5,4, 3,5,4,4,5, 4,4,3,4,5, 5,5,2,4,5, 5,4,5,4,5, 4,5,4,3,5)]
ratings <- cbind(rtr1, rtr2, rtr3, rtr4, rtr5, rtr6)
library(DescTools)
KappaM(ratings, conf.level=0.95)
    kappa    lwr.ci    upr.ci
0.4302445 0.3824725 0.4780165 

### Kendall’s $$W$$ for continuous ordinal ratings and two or more raters

rtr1 <- c(1, 6, 3, 2, 5, 4)
rtr2 <- c(1, 5, 6, 2, 4, 3)
rtr3 <- c(2, 3, 6, 5, 4, 1)
ratings <- cbind(rtr1, rtr2, rtr3)
library(DescTools)
KendallW(ratings, test=TRUE)

Kendall's coefficient of concordance W

data:  ratings
Kendall chi-squared = 8.5238, df = 5, subjects = 6, raters = 3, p-value
= 0.1296
alternative hypothesis: W is greater 0
sample estimates:
W
0.568254 

## Intra-class-correlation for interval-scale ratings and two or more raters

rtr1 <- c(9, 6, 8, 7, 10, 6)
rtr2 <- c(2, 1, 4, 1,  5, 2)
rtr3 <- c(5, 3, 6, 2,  6, 4)
rtr4 <- c(8, 2, 8, 6,  9, 7)
ratings <- cbind(rtr1, rtr2, rtr3, rtr4)
library(DescTools)
ICC(ratings)

Intraclass correlation coefficients
type   est F-val df1 df2    p-val lwr.ci upr.ci
Single_raters_absolute   ICC1 0.166  1.79   5  18 0.164769     NA     NA
Single_random_raters     ICC2 0.290 11.03   5  15 0.000135     NA     NA
Single_fixed_raters      ICC3 0.715 11.03   5  15 0.000135     NA     NA
Average_raters_absolute ICC1k 0.443  1.79   5  18 0.164769     NA     NA
Average_random_raters   ICC2k 0.620 11.03   5  15 0.000135     NA     NA
Average_fixed_raters    ICC3k 0.909 11.03   5  15 0.000135     NA     NA

Number of subjects = 6     Number of raters = 4 

## Detach (automatically) loaded packages (if possible)

try(detach(package:DescTools))