diff --git a/app/controllers/pages_controller.rb b/app/controllers/pages_controller.rb index a22ef97a5..e4fea3828 100644 --- a/app/controllers/pages_controller.rb +++ b/app/controllers/pages_controller.rb @@ -19,10 +19,9 @@ def find_placecal end end + def terms_of_use; end + def robots - robots = File.read(Rails.root.join("config/robots/robots.#{Rails.env}.txt")) - render plain: robots + render plain: current_site.robots end - - def terms_of_use; end end diff --git a/app/controllers/sites_controller.rb b/app/controllers/sites_controller.rb index 3dcd50698..c4802a7f6 100644 --- a/app/controllers/sites_controller.rb +++ b/app/controllers/sites_controller.rb @@ -15,11 +15,6 @@ def index end end - def robots - robots = File.read(Rails.root.join("config/robots/robots.#{Rails.env}.txt")) - render plain: robots - end - private def set_places_to_get_computer_access diff --git a/app/models/site.rb b/app/models/site.rb index c8bf2df52..bfd315df3 100644 --- a/app/models/site.rb +++ b/app/models/site.rb @@ -133,6 +133,20 @@ def og_description tagline && tagline.empty? ? false : tagline end + def robots + config = File.read(Rails.root.join("config/robots/robots.#{Rails.env}.txt")) + + if is_published? + config + else + <<~TXT + #{config} + User-agent: * + Disallow: / + TXT + end + end + class << self # Find the requested Site from information in the rails request object. # diff --git a/config/robots/robots.development.txt b/config/robots/robots.development.txt index f6ca098a0..37b576a4a 100644 --- a/config/robots/robots.development.txt +++ b/config/robots/robots.development.txt @@ -1,5 +1 @@ # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -User-agent: * -Disallow: / diff --git a/config/robots/robots.staging.txt b/config/robots/robots.staging.txt index f6ca098a0..ac1cbe885 100644 --- a/config/robots/robots.staging.txt +++ b/config/robots/robots.staging.txt @@ -1,5 +1,3 @@ # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: User-agent: * -Disallow: / +Disallow: / \ No newline at end of file diff --git a/config/robots/robots.test.txt b/config/robots/robots.test.txt new file mode 100644 index 000000000..d0fb33530 --- /dev/null +++ b/config/robots/robots.test.txt @@ -0,0 +1 @@ +# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file \ No newline at end of file diff --git a/test/integration/robots_integration_test.rb b/test/integration/robots_integration_test.rb new file mode 100644 index 000000000..2e9dfd646 --- /dev/null +++ b/test/integration/robots_integration_test.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'test_helper' + +class RobotsIntegrationTest < ActionDispatch::IntegrationTest + setup do + @published_site = create(:site, is_published: true) + @unpublished_site = create(:site, is_published: false) + end + + test 'robots.txt blocks site if site is unpublished' do + get "http://#{@unpublished_site.slug}.lvh.me:3000/robots.txt" + assert_response 200 + assert_equal forbid_string, response.body + end + + test 'robots.txt has default comment if site is published' do + get "http://#{@published_site.slug}.lvh.me:3000/robots.txt" + assert_response 200 + assert_equal '# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file', response.body + end + + private + + def forbid_string + <<~TXT + # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file + User-agent: * + Disallow: / + TXT + end +end