Skip to content

Commit

Permalink
Organise CSVs and stop needlessly exporting massive dataset (#1271)
Browse files Browse the repository at this point in the history 
* Organise CSVs and stop needlessly exporting massive dataset

* Rewrite data export for users who still have not passed a module
  • Loading branch information
@peterdavidhamilton
peterdavidhamilton authored Jul 19, 2024
1 parent db92d19 commit 23d93a1
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 36 deletions.
23 changes: 12 additions & 11 deletions app/models/data_analysis/users_not_passing.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,25 @@ def column_names

# @return [Array<Hash{Symbol => Mixed}>]
def dashboard
total_users_not_passing_per_module.map do |module_name, count|
Training::Module.live.map do |mod|
{
module_name: module_name,
count: count,
module_name: mod.name,
count: still_failed(mod),
}
end
end

private

# @return [Hash{String => Integer}]
def total_users_not_passing_per_module
Assessment.all
.group(:training_module, :user_id)
.count
.reject { |(module_name, user_id), _| Assessment.all.where(training_module: module_name, user_id: user_id).passed.exists? }
.group_by { |(module_name, _), _| module_name }
.transform_values(&:size)
# @param mod [Training::Module]
# @return [Integer]
def still_failed(mod)
passed_user_ids = Assessment.passed.where(training_module: mod.name).pluck(:user_id)

Assessment.failed
.where(training_module: mod.name)
.where.not(user_id: passed_user_ids)
.pluck(:user_id).uniq.count
end
end
end
Expand Down
51 changes: 27 additions & 24 deletions app/services/dashboard.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,30 +12,33 @@ class Dashboard

# @return [Array<Hash{ Symbol => String }>]
DATA_SOURCES = [
{ model: 'Event', folder: 'eventsdata', file: 'ahoy_events' },
{ model: 'Visit', folder: 'visitsdata', file: 'ahoy_visits' },
{ model: 'User', folder: 'userdata', file: 'users' },
{ model: 'Response', folder: 'useranswers', file: 'responses' },
{ model: 'DataAnalysis::ConfidenceCheckScores', folder: 'useranswers', file: 'confidence_check_scores' },
{ model: 'Assessment', folder: 'userassessments', file: 'assessments' },
{ model: 'DataAnalysis::LocalAuthorityUser', folder: 'localauthorities', file: 'local_authority_users' },
{ model: 'DataAnalysis::AveragePassScores', folder: 'summativequiz', file: 'average_pass_scores' },
{ model: 'DataAnalysis::HighFailQuestions', folder: 'summativequiz', file: 'high_fail_questions' },
{ model: 'DataAnalysis::SettingPassRate', folder: 'summativequiz', file: 'setting_pass_rate' },
{ model: 'DataAnalysis::RolePassRate', folder: 'summativequiz', file: 'role_pass_rate' },
{ model: 'DataAnalysis::UsersNotPassing', folder: 'summativequiz', file: 'users_not_passing_per_module' },
{ model: 'DataAnalysis::ResitsPerUser', folder: 'summativequiz', file: 'resits_per_user' },
{ model: 'DataAnalysis::ModulesPerMonth', folder: 'summativequiz', file: 'modules_per_month' },
{ model: 'DataAnalysis::UserOverview', folder: 'overview', file: 'user_overview' },
{ model: 'DataAnalysis::ModuleOverview', folder: 'overview', file: 'module_overview' },
{ model: 'DataAnalysis::ClosedAccounts', folder: 'overview', file: 'closed_accounts' },
{ model: 'DataAnalysis::UserModuleOrder', folder: 'nonlinear', file: 'user_module_order' },
{ model: 'DataAnalysis::UserModuleCompletion', folder: 'nonlinear', file: 'user_module_completion' },
{ model: 'DataAnalysis::UserModuleCompletionCount', folder: 'nonlinear', file: 'user_module_completions_count' },
{ model: 'DataAnalysis::ReturningUsers', folder: 'nonlinear', file: 'returning_users' },
{ model: 'DataAnalysis::ModuleFeedbackForms', folder: 'feedback', file: 'feedback_forms' },
{ model: 'DataAnalysis::UserFeedbackScores', folder: 'feedback', file: 'feedback_user_scores' },
{ model: 'DataAnalysis::GuestFeedbackScores', folder: 'feedback', file: 'feedback_guest_scores' },
{ model: 'Event', folder: 'events', file: 'events' },

{ model: 'Visit', folder: 'visits', file: 'visits' },

{ model: 'DataAnalysis::UserOverview', folder: 'users', file: 'user_overview' },
{ model: 'User', folder: 'users', file: 'users' },
{ model: 'DataAnalysis::ClosedAccounts', folder: 'users', file: 'closed_accounts' },
{ model: 'DataAnalysis::ReturningUsers', folder: 'users', file: 'returning_users' },
{ model: 'DataAnalysis::LocalAuthorityUser', folder: 'users', file: 'local_authority_users' },

{ model: 'Assessment', folder: 'training', file: 'assessments' },
{ model: 'DataAnalysis::ConfidenceCheckScores', folder: 'training', file: 'confidence_check_scores' },
{ model: 'DataAnalysis::AveragePassScores', folder: 'training', file: 'average_pass_scores' },
{ model: 'DataAnalysis::HighFailQuestions', folder: 'training', file: 'high_fail_questions' },
{ model: 'DataAnalysis::SettingPassRate', folder: 'training', file: 'setting_pass_rate' },
{ model: 'DataAnalysis::RolePassRate', folder: 'training', file: 'role_pass_rate' },
{ model: 'DataAnalysis::UsersNotPassing', folder: 'training', file: 'assessment_failures' },
{ model: 'DataAnalysis::ResitsPerUser', folder: 'training', file: 'resits_per_user' },
{ model: 'DataAnalysis::ModulesPerMonth', folder: 'training', file: 'modules_per_month' },
{ model: 'DataAnalysis::ModuleOverview', folder: 'training', file: 'module_overview' },
{ model: 'DataAnalysis::UserModuleOrder', folder: 'training', file: 'module_order' },
{ model: 'DataAnalysis::UserModuleCompletion', folder: 'training', file: 'module_completion' },
{ model: 'DataAnalysis::UserModuleCompletionCount', folder: 'training', file: 'module_completions_count' },

{ model: 'DataAnalysis::GuestFeedbackScores', folder: 'feedback', file: 'guest_feedback' },
{ model: 'DataAnalysis::UserFeedbackScores', folder: 'feedback', file: 'course_feedback' },
{ model: 'DataAnalysis::ModuleFeedbackForms', folder: 'feedback', file: 'module_feedback' },
].freeze

# @return [String] 30-06-2022-09-30
Expand Down
16 changes: 15 additions & 1 deletion spec/models/data_analysis/users_not_passing_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,32 @@
[
{
module_name: 'alpha',
count: 1,
count: 2,
},
{
module_name: 'bravo',
count: 0,
},
{
module_name: 'charlie',
count: 0,
},
]
end

let(:user_1) { create :user, :registered }
let(:user_2) { create :user, :registered }
let(:user_3) { create :user, :registered }

before do
create :assessment, :failed, user: user_1
create :assessment, :failed, user: user_1
create :assessment, :failed, user: user_1

create :assessment, :failed, user: user_2
create :assessment, :passed, user: user_2

create :assessment, :failed, user: user_3
end

it_behaves_like 'a data export model'
Expand Down

0 comments on commit 23d93a1

Please sign in to comment.