Skip to content

Commit

Permalink
added new references
Browse files Browse the repository at this point in the history
  • Loading branch information
ManuelRausch committed Apr 4, 2024
1 parent 1e866a8 commit baf69a6
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 4 deletions.
67 changes: 67 additions & 0 deletions paper.bib
Original file line number Diff line number Diff line change
Expand Up @@ -286,3 +286,70 @@ @misc{hellmann_confidence_2024
langid = {english},
file = {Hellmann et al. - 2023 - Confidence is influenced by evidence accumulation .pdf:C\:\\Users\\mru\\Zotero\\storage\\XRR256WT\\Hellmann et al. - 2023 - Confidence is influenced by evidence accumulation .pdf:application/pdf},
}

@online{shekhar_nature_2020,
title = {The nature of metacognitive inefficiency in perceptual decision making},
url = {https://osf.io/s8fnb/},
abstract = {Hosted on the Open Science Framework},
author = {Shekhar, Medha and Rahnev, Dobromir},
urldate = {2024-04-04},
date = {2020-04-03},
note = {Publisher: {OSF}},
file = {Snapshot:C\:\\Users\\mru\\Zotero\\storage\\DFGNZ5E7\\s8fnb.html:text/html},
}

@online{shekhar_how_2022,
title = {How do humans give confidence? A comprehensive comparison of process models of metacognition},
url = {https://osf.io/g8f9x/},
shorttitle = {How do humans give confidence?},
abstract = {Hosted on the Open Science Framework},
author = {Shekhar, Medha and Rahnev, Dobromir},
urldate = {2024-04-04},
date = {2022-03-30},
note = {Publisher: {OSF}},
file = {Snapshot:C\:\\Users\\mru\\Zotero\\storage\\7Y9RWYDA\\g8f9x.html:text/html},
}

@online{rausch_full_2017,
title = {Full material to "Confidence in masked orientation judgments is informed by both evidence and visibility"},
url = {https://osf.io/ty4h8/},
abstract = {Hosted on the Open Science Framework},
author = {Rausch, Manuel and Zehetleitner, Michael},
urldate = {2024-04-04},
date = {2017-04-07},
note = {Publisher: {OSF}},
file = {Snapshot:C\:\\Users\\mru\\Zotero\\storage\\BCV2C5UX\\ty4h8.html:text/html},
}

@online{rausch_full_2022,
title = {Full material to "Measures of metacognitive efficiency across cognitive models of decision confidence"},
url = {https://osf.io/72uds/},
abstract = {Abstract
Meta-d’/d’ has become the quasi-gold standard to quantify metacognitive efficiency because meta-d’/d’ is widely believed to control for discrimination performance, discrimination criteria, and confidence criteria even without the assumption of a specific generative model underlying confidence judgments. Using simulations, we demonstrate that meta-d’/d’ is not as model-free as previously thought: Only when we simulated data using a generative model of confidence according to which the evidence underlying confidence judgements is sampled independently from the evidence utilized in the choice process from a truncated Gaussian distribution, meta-d’/d’ was unaffected by discrimination performance, discrimination task criteria, and confidence criteria. According to five alternative generative models of confidence, there exist at least some combination of parameters where meta-d’/d’ is affected by discrimination performance, discrimination criteria and confidence criteria. A simulation using empirically fitted parameter sets showed that the magnitude of the correlation between meta-d’/d’ and discrimination performance, discrimination task criteria, and confidence criteria depends heavily on the generative model and the specific parameter and varies between negligibly small and very large. These simulations imply that a difference in meta-d’/d’ between conditions does not necessarily reflect a difference in metacognitive efficiency but might as well be caused by a difference in discrimination performance, discrimination task criterion, or confidence criteria.
Flies
This files provide all code necessary to replicate the results and Figures.
All analyses were performed using version 4.0.3 running on Windows 10, using the R packages tidyverse 1.3.0, plyr 1.8.7, ggplot2 3.3.3, rjags 4.13, coda 0.19.4., R.utils 2.12.0, {gridExtra} 2.3, Rmisc 1.5.1, snow 0.4.0 and {doSNOW} 1.0.20.
To replicate Simulation 1, download all files to your local computer and place all files in the same directory. Then, run the file "Mratio\_byModels\_analysis.R.R". The file Mratio\_Plotting\_v2023.R contains the code to plot the data. The file "Mratio\_PlotCognitiveModelsAndMethods.{RData}" contains all the Figures.
To replicate Simulation 2, run the file "Reanalyze\_RouaultEtAl2018Exp2\_ConfidenceModels.R". Depending on your hardware, this might take a while. The results are stored in the files "Correlations\_RouaultEtAlExp2\_ConfidenceModels.{RData}", "Simulations\_RouaultEtAlExp2\_ConfidenceModels.{RData}", and "{ModelFits}\_RouaultEtAlExp2\_ConfidenceModels.{RData}".
Hosted on the Open Science Framework},
author = {Rausch, Manuel},
urldate = {2024-04-04},
date = {2022-10-21},
note = {Publisher: {OSF}},
file = {Snapshot:C\:\\Users\\mru\\Zotero\\storage\\E8PLF38I\\72uds.html:text/html},
}

@online{rausch_full_2018,
title = {Full material to "Cognitive modelling reveals distinct electrophysiological markers of decision confidence and error monitoring"},
url = {https://osf.io/93weg/},
author = {Rausch, Manuel and Zehetleitner, Michael and Steinhauser, Marco and Maier, Martin E},
urldate = {2024-04-04},
date = {2018},
file = {OSF | Full material to "Cognitive modelling reveals distinct electrophysiological markers of decision confidence and error monitoring":C\:\\Users\\mru\\Zotero\\storage\\LBPH7N8D\\93weg.html:text/html},
}
8 changes: 4 additions & 4 deletions paper.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,16 @@ bibliography: paper.bib

# Summary

We present the statConfR package for R, which allows researchers to conveniently fit and compare nine different static models of decision confidence applicable to binary discrimination tasks with confidence ratings: the signal detection rating model [@Green1966], the Gaussian noise model[@Maniscalco2016], the independent Gaussian model [@Rausch2017], the weighted evidence and visibility model [@Rausch2018], the lognormal noise model [@Shekhar2020a], the lognormal weighted evidence and visibility model [@shekhar_how_2023], the independent truncated Gaussian model [@rausch_measures_2023] based on the meta-d′/d′ method [@Maniscalco2012; @Maniscalco2014], and the independent truncated Gaussian model based on the Hmetad method [@Fleming2017a]. In addition, the statConfR package provides functions to estimate meta-d′/d′, the most widely-used measure of metacognitive efficiency, allowing both @Maniscalco2012 and @Fleming2017a's model specification.
We present the statConfR package for R, which allows researchers to conveniently fit and compare nine different static models of decision confidence applicable to binary discrimination tasks with confidence ratings: the signal detection rating model [@Green1966], the Gaussian noise model[@Maniscalco2016], the independent Gaussian model [@Rausch2017], the weighted evidence and visibility model [@Rausch2018], the lognormal noise model [@Shekhar2020a], the lognormal weighted evidence and visibility model [@shekhar_how_2023], the independent truncated Gaussian model [@rausch_measures_2023] based on the meta-d′/d′ method [@Maniscalco2012; @Maniscalco2014], and the independent truncated Gaussian model based on the Hmetad method [@Fleming2017a]. In addition, the statConfR package provides functions for estimating meta-d′/d′, the most widely-used measure of metacognitive efficiency, allowing both @Maniscalco2012's and @Fleming2017a's model specification.

# Statement of need

Cognitive models of confidence are currently used implicitly and explicitly across a wide range of research areas in the Cognitive Sciences: In perception research, confidence judgements can be used to quantify perceptual sensitivity based on receiver operating characteristics [@egan_operating_1959], a method that relies on the signal detection rating model [@Green1966; @hautus_detection_2021]. In metacognition research, the most popular measure of metacognitive accuracy, the meta-d′/d′ method [@Maniscalco2012; @Maniscalco2014], depends on the independent truncated Gaussian model [@rausch_measures_2023]. Finally, models of confidence have become a flourishing research topic itself [@boundy-singer_confidence_2022; @Desender2021; @guggenmos_reverse_2022; @hellmann_confidence_2024; @hellmann_simultaneous_2023; @pereira_evidence_2021; @Rausch2018; @Rausch2020; @Shekhar2020a; @shekhar_how_2023]. However, up to date, too few studies have compared models of confidence empirically [@Rausch2018; @Rausch2020; @rausch_measures_2023; @Shekhar2020a, @shekhar_how_2023], which is why there is still no consensus about the computational principles underlying confidence judgments [@rahnev_consensus_2022]. This is problematic because meta-d′/d′ can be biased by discrimination sensitivity, discrimination criteria, and/or confidence criteria if the generative model underlying the data is not independent truncated Gaussian model [@rausch_measures_2023]. Likewise, receiver operating characteristics in rating experiments are appropriate measures of discrimination sensitivity only if the assumptions of the signal detection rating model are correct [@Green1966; @hautus_detection_2021].
Cognitive models of confidence are currently used implicitly and explicitly in a wide range of research areas in the cognitive sciences: In perception research, confidence judgments can be used to quantify perceptual sensitivity based on receiver operating characteristics [@egan_operating_1959], a method based on the signal detection rating model [@Green1966; @hautus_detection_2021]. In metacognition research, the most popular measure of metacognitive accuracy, the meta-d′/d′ method [@Maniscalco2012; @Maniscalco2014], implicitly relies on the independent truncated Gaussian model [@rausch_measures_2023]. Finally, confidence models have become a flourishing research topic in their own right [@boundy-singer_confidence_2022; @Desender2021; @guggenmos_reverse_2022; @hellmann_confidence_2024; @hellmann_simultaneous_2023; @pereira_evidence_2021; @Rausch2018; @Rausch2020; @Shekhar2020a; @shekhar_how_2023]. However, too few studies have empirically compared different confidence models [@Rausch2018; @Rausch2020; @rausch_measures_2023; @Shekhar2020a, @shekhar_how_2023], so there is still no consensus about the computational principles underlying confidence judgments [@rahnev_consensus_2022]. This is problematic because meta-d′/d′ can be biased by discrimination sensitivity, discrimination criteria, and/or confidence criteria if the generative model underlying the data is not the independent truncated Gaussian model [@rausch_measures_2023]. Likewise, receiver operating characteristics in rating experiments are only appropriate measures of discrimination sensitivity if the assumptions of the signal detection rating model are correct [@Green1966; @hautus_detection_2021].

At the time of writing this manuscript, statConfR is the only available package for an open software that allows researchers to fit a set of static models of decision confidence. The ReMeta-toolbox for MATLAB provides functions to fit a variety of different confidence models too [@guggenmos_reverse_2022], but several important models such as the independent truncated Gaussian model are missing. Previous studies modelling confidence have made their analysis scripts freely available at the osf website [REFERENCES MISSING], but these analysis scripts are often tailored to specific experiments and require time and effort to adapt to new experiments. In addition, the documentation of these scripts is not always sufficient to allow researchers without export knowledge in cognitive modelling to adapt these scripts for their own data sets. Finally, the lognormal noise model and the lognormal weighted evidence and visibility model have been previously available only in MATLAB, which is why statConfR makes these confidence models available to researchers who do not have access to MATLAB.
At the time of writing, statConfR is the only available package for an open software that allows researchers to fit a set of static models of decision confidence. The ReMeta toolbox provides functions for MATLAB to also fit a variety of different confidence models [@guggenmos_reverse_2022], but some important models such as the independent truncated Gaussian model are missing. Previous studies modelling confidence have made their analysis scripts freely available on the OSF website [@rausch_full_2017; @rausch_full_2018; @rausch_full_2022; @shekhar_nature_2020-1; @shekhar_how_2022], but these analysis scripts are often tailored to specific experiments and require time and effort to adapt to new experiments. In addition, the documentation of these scripts is not always sufficient to be used without export knowledge in cognitive modelling. Finally, the lognormal noise model and the lognormal weighted evidence and visibility model were previously only available in MATLAB, so statConfR makes these confidence models available to researchers who do not have access to MATLAB.

# Acknowledgements

This research was in part supported by grants RA2988/3-1 and RA2988/4-1 by the Deutsche Forschungsgemeinschaft. The funders had no role in study design, data collection, analysis, decision to publish, or preparation of the manuscript. We have no conflicts of interest to disclose.
This research was in part supported by grants RA2988/3-1 and RA2988/4-1 by the Deutsche Forschungsgemeinschaft.

# References

0 comments on commit baf69a6

Please sign in to comment.