DEV-1242 Add report export functionality to OTIS (#222)

* DEV-1242 Add report export functionality to OTIS
- Add `popper_js` gem to add support for Bootstrap dropdown menus.
- Move institutions CSV download button to top of page.
- Move users CSV download button to top of page and add option to exclude ATRS users.
- Add `role_filter` option to `ht_users` controller for CSV requests.
* TIDY: replace obsolete 'docker-compose' invocations (addresses issue #218)
* Address CVE-2024-39908 in rexml gem

Gemfile

@@ -81,6 +81,7 @@ gem "tzinfo-data", platforms: [:mingw, :mswin, :x64_mingw, :jruby]
 #
 ##############################################################################
 gem "autoprefixer-rails"
+gem "popper_js"
 gem "bootstrap-sass"
 gem "jquery-rails"
 gem "ckeditor"
@@ -107,6 +108,8 @@ gem "sequel", "5.52.0"
 group :development, :test do
   gem "byebug"
   gem "standard"
+  # parser should be >= the current Ruby version to avoid warnings
+  gem "parser", ">= 3.1.6"
   gem "pry"
   gem "pry-byebug", ">= 3.9.0"
   gem "sqlite3"

Gemfile.lock

@@ -226,8 +226,10 @@ GEM
       version_gem (~> 1.1)
     orm_adapter (0.5.0)
     parallel (1.22.1)
-    parser (3.1.2.1)
+    parser (3.3.4.0)
       ast (~> 2.4.1)
+      racc
+    popper_js (2.11.8)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -282,8 +284,8 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     regexp_parser (2.6.0)
-    rexml (3.2.8)
-      strscan (>= 3.0.9)
+    rexml (3.3.2)
+      strscan
     rubocop (1.35.1)
       json (~> 2.3)
       parallel (~> 1.10)
@@ -411,6 +413,8 @@ DEPENDENCIES
   net-imap
   net-pop
   net-smtp
+  parser (>= 3.1.6)
+  popper_js
   pry
   pry-byebug (>= 3.9.0)
   puma (~> 5.6)

README.md

@@ -10,14 +10,14 @@
 ```
 $ git clone https://github.com/hathitrust/otis.git
 $ cd otis
-$ docker-compose build
-$ docker-compose run web bundle install
+$ docker compose build
+$ docker compose run --rm web bundle install
 ```
 
 ### 2. Trying it out
 
 ```
-docker-compose up -d web
+docker compose up -d web
 ```
 
 Development mode uses mysql via Docker with generated data from the `db:seed`
@@ -32,24 +32,24 @@ administrative power.
 ### 3. Running tests
 
 ```
-docker-compose run test
+docker compose run --rm test
 ```
 
 To enable W3C HTML validation of OTIS pages, use the following.
 These tests are not run by default since they rely on an external service.
 
 ```
-docker-compose run -e W3C_VALIDATION=1 test
+docker compose run --rm -e W3C_VALIDATION=1 test
 ```
 
 To run a single test class use an invocation along these lines:
 
 ```
-docker-compose run test bundle exec ruby -I test test/controllers/ht_users_controller_test.rb
+docker compose run --rm test bundle exec ruby -I test test/controllers/ht_users_controller_test.rb
 ```
 
 System tests, as usual, are not run by default.
 
 ```
-docker-compose run system-test
+docker compose run --rm system-test
 ```

app/assets/javascripts/application.js

@@ -11,6 +11,7 @@
 // about supported directives.
 //
 //= require jquery3
+//= require popper
 //= require bootstrap
 //= require rails-ujs
 //= require activestorage

app/controllers/ht_institutions_controller.rb

@@ -38,7 +38,10 @@ def index
     @other_institutions = HTInstitution.other.order("name").map { |i| presenter i }
     respond_to do |format|
       format.html
-      format.csv { send_data institutions_csv }
+      format.csv do
+        file_name = (params[:file_name] || "ht_institutions") + ".csv"
+        send_data institutions_csv, filename: file_name
+      end
     end
   end
 

app/controllers/ht_users_controller.rb

@@ -13,7 +13,10 @@ def index
     @all_users = users.map { |u| presenter u }
     respond_to do |format|
       format.html
-      format.csv { send_data users_csv }
+      format.csv do
+        file_name = (params[:file_name] || "ht_users") + ".csv"
+        send_data users_csv, filename: file_name
+      end
     end
   end
 
@@ -65,6 +68,9 @@ def users_csv
     CSV.generate do |csv|
       csv << @all_users.first.csv_cols
       @all_users.each do |user|
+        if params[:role_filter]&.include?(user.role)
+          next
+        end
         csv << user.csv_vals
       end
     end

app/views/ht_institutions/index.html.erb

@@ -2,6 +2,9 @@
 
   <% fields = HTInstitutionPresenter::INDEX_FIELDS %>
 
+  <%= link_to t(".download_csv"), ht_institutions_url(format: :csv), class: 'btn btn-info' %>
+  <br/>
+
   <h2 class="pull-left"><%= t ".enabled_institutions" %></h2>
 
   <table id="active_institutions" class="table table-striped" data-toggle="table" data-height="460" data-virtual-scroll="true"
@@ -45,10 +48,6 @@
     <% end %>
   </table>
 
-  <br />
-  <%= link_to t(".download_csv"), ht_institutions_url(format: :csv), class: 'btn btn-info' %>
-  <br />
-
   <% if can?(:create, HTInstitution) %>
     <h2><%= t ".add" %></h2>
 

app/views/ht_users/index.html.erb

@@ -4,6 +4,22 @@
   <% fields = HTUserPresenter::INDEX_FIELDS %>
   <%= form_tag(ht_approval_requests_path, method: :post) do %>
 
+  <div class="dropdown">
+    <button class="btn btn-info dropdown-toggle" type="button" id="download-menu"
+      data-toggle="dropdown" aria-haspopup="true" aria-expanded="true">
+      <%= t(".download_csv") %>
+      <span class="caret"></span>
+    </button>
+    <ul class="dropdown-menu" aria-labelledby="download-menu">
+      <li>
+        <%= link_to t(".download_csv_non_atrs"), ht_users_url(format: :csv, role_filter: [:ssd, :ssdproxy], file_name: "non_atrs_users") %>
+      </li>
+      <li>
+        <%= link_to t(".download_csv_all"), ht_users_url(format: :csv, file_name: "all_users") %>
+      </li>
+    </ul>
+  </div>
+
   <h2 class="pull-left"><%= t ".active_users" %></h2>
   <table id="active_users" class="table table-striped" data-toggle="table" data-height="460" data-virtual-scroll="true"
          data-search="true" data-show-search-clear-button="true">
@@ -37,7 +53,7 @@
   <% if can?(:edit, HTUser) %>
     <%= button_tag t(".renew_selected_users"), type: 'submit', name: 'submit_renewals', class: 'btn btn-primary' %>
   <% end %>
-  <%= link_to t(".download_csv"), ht_users_url(format: :csv), class: 'btn btn-info' %>
+
   <% end # form_tag %>
 
   <h2 class="pull-left"><%= t ".expired_users" %></h2>

config/locales/en.yml

@@ -394,6 +394,8 @@ en:
       active_users: Active Users
       create_approval_requests: Create Approval Requests
       download_csv: Download CSV
+      download_csv_all: All Users
+      download_csv_non_atrs: Non-ATRS Users
       expired_users: Expired Users
       renew_selected_users: Renew Selected Users
       select: Select

config/locales/ja.yml

@@ -394,6 +394,8 @@ ja:
       active_users: アクティブユーザー
       create_approval_requests: 承認リクエストを作成する
       download_csv: CSVをダウンロード
+      download_csv_all: 全てのユーザー
+      download_csv_non_atrs: 非ATRSのユーザー
       expired_users: 期限切れのユーザー
       renew_selected_users: 選択したユーザーを更新する
       select: 選択

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
 
   web:

docker-compose.yml.arm64

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
 
   web:

test/controllers/ht_users_controller_test.rb

@@ -266,7 +266,7 @@ def setup
       expire_type: "expiresannually", iprestrict: "any",
       mfa: false, identity_provider: "http://example.com", inst_id: "X")
     @user1.save!
-    @user2 = create(:ht_user, :expired, userid: "y@z")
+    @user2 = create(:ht_user, :expired, userid: "y@z", role: :crms)
   end
 
   test "export list of all users as CSV" do
@@ -280,6 +280,18 @@ def setup
     assert_match "a@b,A B,c@d,e@f,g@h,i@j,staff,ssd,total,2020-01-01 00:00:00 UTC," \
                  "expiresannually,^.*$,false,http://example.com,X,Y", @response.body
   end
+
+  test "export list of non-ATRS users as CSV" do
+    sign_in!
+    get ht_users_url format: :csv, role_filter: [:ssd, :ssdproxy]
+    assert_equal 2, @response.body.lines.count
+    assert_equal @response.body.lines[0].strip,
+      "userid,displayname,email,activitycontact,approver," \
+      "authorizer,usertype,role,access,expires,expire_type," \
+      "iprestrict,mfa,identity_provider,inst_id,inst_name", @response.body
+    roles = @response.body.lines[1..].map { |line| line.split(",")[7] }
+    refute roles.include? :ssd
+  end
 end
 
 class HTUsersControllerRolesTest < ActionDispatch::IntegrationTest