diff --git a/Dockerfile b/Dockerfile index 1fddd9b44e..52e414396c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,22 +53,7 @@ WORKDIR ${APP_HOME} EXPOSE 3000 -CMD /filebeat/filebeat -c /filebeat/filebeat.yml & bundle exec rails server - -# Download and install filebeat for sending logs to logstash -ENV FILEBEAT_VERSION=7.6.2 -ENV FILEBEAT_DOWNLOAD_PATH=/tmp/filebeat.tar.gz -ENV FILEBEAT_CHECKSUM=482304509aed80db78ef63a0fed88e4453ebe7b11f6b4ab3168036a78f6a413e2f6a5c039f405e13984653b1a094c23f7637ac7daf3da75a032692d1c34a9b65 - -RUN curl https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz -o ${FILEBEAT_DOWNLOAD_PATH} && \ - [ "$(sha512sum ${FILEBEAT_DOWNLOAD_PATH})" = "${FILEBEAT_CHECKSUM} ${FILEBEAT_DOWNLOAD_PATH}" ] && \ - tar xzvf ${FILEBEAT_DOWNLOAD_PATH} && \ - rm ${FILEBEAT_DOWNLOAD_PATH} && \ - mv filebeat-${FILEBEAT_VERSION}-linux-x86_64 /filebeat && \ - rm -f /filebeat/filebeat.yml - -# Copy our local filebeat config to the installation -COPY filebeat.yml /filebeat/filebeat.yml +CMD bundle exec rails server # Copy dependencies (relying on dependencies using the same base image as this) COPY --from=dependencies ${DEPS_HOME}/Gemfile ${APP_HOME}/Gemfile @@ -96,7 +81,7 @@ RUN DFE_SIGN_IN_API_CLIENT_ID= \ SUPPRESS_DFE_ANALYTICS_INIT= \ bundle exec rake assets:precompile -RUN chown -hR appuser:appgroup ${APP_HOME} /filebeat +RUN chown -hR appuser:appgroup ${APP_HOME} USER appuser diff --git a/bin/start-worker b/bin/start-worker index 847fa25529..9c65825b26 100755 --- a/bin/start-worker +++ b/bin/start-worker @@ -1,6 +1,4 @@ #!/bin/bash echo "Starting workers..." -/filebeat/filebeat -c /filebeat/filebeat.yml \ - & bundle exec bin/delayed_job start -n "$WORKER_COUNT" \ - & tail -f "log/$RAILS_ENV.log" +bundle exec bin/delayed_job run -n "$WORKER_COUNT" diff --git a/config/environments/development.rb b/config/environments/development.rb index e98dbb2339..d7b3cb6f23 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -87,4 +87,9 @@ # Add '127.0.0.1 ecp.test' to your /etc/hosts to use this over localhost. # This allows you to use https://ecp.test:3000/ in your browser. config.hosts << "ecp.test" + + # https://technical-guidance.education.gov.uk/infrastructure/monitoring/logit/#ruby-on-rails + config.log_level = :debug # Or :info + config.log_format = :color # Console colorised non-json output + config.semantic_logger.backtrace_level = :debug # Show file and line number (expensive: not for production) end diff --git a/config/environments/production.rb b/config/environments/production.rb index 6574840857..21b9b7fd6d 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -47,8 +47,11 @@ # information to avoid inadvertent exposure of personally identifiable information (PII). config.log_level = :info - # Prepend all log lines with the following tags. - config.log_tags = [:request_id] + # https://technical-guidance.education.gov.uk/infrastructure/monitoring/logit/#ruby-on-rails + config.log_format = :json # For parsing in Logit + config.rails_semantic_logger.add_file_appender = false # Don't log to file + config.active_record.logger = nil # Don't log SQL + config.rails_semantic_logger.filter = proc { |log| log.name != "DfE::Analytics::SendEvents" } # Use a different cache store in production. # config.cache_store = :mem_cache_store @@ -89,16 +92,6 @@ # require "syslog/logger" # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new "app-name") - if ENV["RAILS_LOG_TO_STDOUT"].present? - logger = ActiveSupport::Logger.new($stdout) - logger.formatter = config.log_formatter - config.logger = ActiveSupport::TaggedLogging.new(logger) - - $stdout.sync = true - config.rails_semantic_logger.add_file_appender = false - config.semantic_logger.add_appender(io: $stdout, formatter: config.rails_semantic_logger.format) - end - # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false end diff --git a/config/initializers/semantic_logger.rb b/config/initializers/semantic_logger.rb index 96bd559173..97dcc7763f 100644 --- a/config/initializers/semantic_logger.rb +++ b/config/initializers/semantic_logger.rb @@ -1,3 +1,10 @@ +# https://technical-guidance.education.gov.uk/infrastructure/monitoring/logit/#ruby-on-rails Rails.application.configure do - config.semantic_logger.application = "Claim Additional Payments for Teaching" + config.semantic_logger.application = "" # This is added by logstash from its tags + config.log_tags = [:request_id] # Prepend all log lines with the following tags +end + +unless Rails.env.test? + SemanticLogger.add_appender(io: $stdout, level: Rails.application.config.log_level, formatter: Rails.application.config.log_format) + Rails.application.config.logger.info("Application logging to STDOUT") end diff --git a/docs/logging.md b/docs/logging.md index e3c5b58331..dc4d8ceee9 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -1,33 +1,24 @@ # Logging -# Application-level logging in logit.io +The application logs to STDOUT in JSON format using [Semantic Logger](https://github.com/reidmorrison/rails_semantic_logger). -The application sends all Rails logs to [logit.io](https://logit.io/). This -gives us a hosted ELK stack. ELK stands for Elasticsearch, Logstash, Kibana. -It’s a open-source stack frequently used for log aggregation and visualisation. +# AKS/container logs -Kibana is the tool that you’ll use to explore the logs. +Logs are available via the standard Azure/AKS tooling. -To view the production logs in Kibana: - -1. Log in to logit.io. -2. Click on the “DfE Claim” account. -3. Find the “Production” stack and click “Launch Kibana”. - -## Missing logs in logit.io +The `az` Azure command line tool and `kubectl` have some useful commands for interacting with +logs. -At the time of writing (2020-03-26), we have a problem where some log messages -seem to be missing from logit.io. We are trying to fix this in -[this Trello card](https://trello.com/c/JN44De4l/1330-understand-why-expected-logs-arent-in-logit). +```sh +kubectl -n srtl-test get pods +kubectl -n srtl-test logs claim-additional-payments-for-teaching-test-web-123456 +``` -# Azure logs +You will need PIM elevated privileges to view logs for production. -We should aim for logit.io to be the single place developers need to look to -find logs. However, it’s useful to know about some of the logs that are -available from Azure. +# Logit.io -The `az` Azure command line tool has some useful commands for interacting with -logs. +Logs are shipped to [Logit.io](https://logit.io/) automatically by AKS. You will need an account to access this service. ## App Service @@ -38,27 +29,6 @@ There is some about the logs available for an App Service. Here we go over some of the ways to extract these logs. -### Docker logs - -To view Docker logs in production, you will need to elevate your privileges -using a [PIM request](privileged-identity-management-requests.md). - -You can use the `az webapp log` commands to tail and download the logs. - -Here’s an example command that will live tail the container logs and Docker host -logs, for `production`: - -``` -az webapp log tail --name s118p01-app-as --resource-group s118p01-app --subscription s118-teacherpaymentsservice-production -``` - -Or, you can download them with a browser: - -- production Docker logs from the instances currently in the App Service: - https://s118p01-app-as.scm.azurewebsites.net/api/logs/docker -- production Docker logs from current and previous instances: - https://s118d01-app-as.scm.azurewebsites.net/api/vfs/LogFiles/ - ### Application Insights The Rails application uses the `application_insights` gem, which sends @@ -72,22 +42,3 @@ To view these logs: 3. Click “Logs” under “Monitoring” on the left. 4. You can now execute queries. For example, to view all requests, type `requests` in the query field, and click Run. - -## Container Instances - -The container instances perform other tasks like running a background job -worker. The logs are lost after a deploy. You don’t need PIM-elevated privileges -to view the container instance logs. - -To view the logs for a container instance, we can use `az container logs`: - -``` -az container logs --name s118p01-app-worker-aci --resource-group s118p01-app --subscription s118-teacherpaymentsservice-production -``` - -Or, you can view them in a browser: - -1. Visit `https://portal.azure.com`. -2. Search for `s118p01-app-worker-aci`. -3. Click “Containers” on the left. -4. Click “Logs”. diff --git a/filebeat.yml b/filebeat.yml deleted file mode 100644 index 376aaaef85..0000000000 --- a/filebeat.yml +++ /dev/null @@ -1,104 +0,0 @@ -###################### Filebeat Configuration ######################### - -#=========================== Filebeat inputs ============================= - -filebeat.inputs: - -# Each - is an input. Most options can be set at the input level, so -# you can use different inputs for various configurations. -# Below are the input specific configurations. - -- type: log - enabled: true - paths: - - /app/log/*.json - encoding: utf-8 - ignore_older: 3h - json.keys_under_root: true - json.add_error_key: true - - -#============================= Filebeat modules =============================== - -filebeat.config.modules: - # Glob pattern for configuration loading - # path: ${path.config}/modules.d/*.yml - - # Set to true to enable config reloading - # reload.enabled: true - - # Period on which files under path should be checked for changes - # reload.period: 10s - #filebeat.modules: - # - module: system - - -#================================ General ===================================== - -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. -#tags: ["service-X", "web-tier"] - -# Optional fields that you can specify to add additional information to the -# output. -fields: - env: "${ENVIRONMENT_NAME}" - -#================================ Outputs ===================================== - -# Configure what output to use when sending the data collected by the beat. - -#----------------------------- Logstash output -------------------------------- -output.logstash: - # The Logstash hosts - hosts: ["${LOGSTASH_HOST}:${LOGSTASH_PORT}"] - loadbalance: true - ssl.enabled: true - - # Optional SSL. By default is off. - # List of root certificates for HTTPS server verifications - #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication - #ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key - #ssl.key: "/etc/pki/client/cert.key" - -#================================ Processors ===================================== - -# Configure processors to enhance or manipulate events generated by the beat. - -#processors: -# - add_host_metadata: ~ -# - add_cloud_metadata: ~ - -#================================ Logging ===================================== - -# Sets log level. The default log level is info. -# Available log levels are: error, warning, info, debug -#logging.level: debug - -# At debug level, you can selectively enable logging only for some components. -# To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publish", "service". -#logging.selectors: ["*"] - -#============================== Xpack Monitoring =============================== -# filebeat can export internal metrics to a central Elasticsearch monitoring -# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The -# reporting is disabled by default. - -# Set to true to enable the monitoring reporter. -#xpack.monitoring.enabled: false - -# Uncomment to send the metrics to Elasticsearch. Most settings from the -# Elasticsearch output are accepted here as well. Any setting that is not set is -# automatically inherited from the Elasticsearch output configuration, so if you -# have the Elasticsearch output configured, you can simply uncomment the -# following line. -#xpack.monitoring.elasticsearch: diff --git a/terraform/application/application.tf b/terraform/application/application.tf index a9436e58bf..d4afd87ec6 100644 --- a/terraform/application/application.tf +++ b/terraform/application/application.tf @@ -40,6 +40,8 @@ module "web_application" { command = var.startup_command replicas = var.web_replicas + + enable_logit = var.enable_logit } module "worker_application" { @@ -60,4 +62,6 @@ module "worker_application" { command = var.worker_command replicas = var.worker_replicas + + enable_logit = var.enable_logit } diff --git a/terraform/application/config/review.tfvars.json b/terraform/application/config/review.tfvars.json index 5d80a8ed97..496c0b3474 100644 --- a/terraform/application/config/review.tfvars.json +++ b/terraform/application/config/review.tfvars.json @@ -5,5 +5,6 @@ "deploy_azure_backing_services": false, "enable_postgres_ssl": false, "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"], - "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"] + "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"], + "enable_logit": true } diff --git a/terraform/application/config/test.tfvars.json b/terraform/application/config/test.tfvars.json index f3710db414..f367d83ebc 100644 --- a/terraform/application/config/test.tfvars.json +++ b/terraform/application/config/test.tfvars.json @@ -9,5 +9,6 @@ "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"], "enable_monitoring": true, "statuscake_contact_groups": [195955, 282453], - "external_url": "https://claim-additional-payments-for-teaching-test-web.test.teacherservices.cloud/healthcheck" + "external_url": "https://claim-additional-payments-for-teaching-test-web.test.teacherservices.cloud/healthcheck", + "enable_logit": true } diff --git a/terraform/application/variables.tf b/terraform/application/variables.tf index 73b26c8c1c..4453323375 100644 --- a/terraform/application/variables.tf +++ b/terraform/application/variables.tf @@ -77,6 +77,12 @@ variable "postgres_flexible_server_sku" { variable "postgres_enable_high_availability" { default = false } +variable "enable_logit" { + type = bool + default = false + description = "A boolean to indicate whether to enable sending container logs to logit.io" + nullable = false +} locals { postgres_ssl_mode = var.enable_postgres_ssl ? "require" : "disable"