diff --git a/.env.test b/.env.test index 871f85d13642ae..e247a2e952ebcf 100644 --- a/.env.test +++ b/.env.test @@ -4,4 +4,7 @@ NODE_ENV=production LOCAL_DOMAIN=cb6e6126.ngrok.io LOCAL_HTTPS=true # Elasticsearch +ES_ENABLED=false +ES_HOST=localhost +ES_PORT=9200 ES_PREFIX=test diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml index cccaad296d1212..dfd217a5e7d80a 100644 --- a/.github/workflows/test-ruby.yml +++ b/.github/workflows/test-ruby.yml @@ -259,6 +259,19 @@ jobs: ports: - 6379:6379 + search: + image: ${{ matrix.search-image }} + env: + discovery.type: single-node + xpack.security.enabled: false + options: >- + --health-cmd "curl http://localhost:9200/_cluster/health" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 9200:9200 + env: DB_HOST: localhost DB_USER: postgres @@ -300,32 +313,6 @@ jobs: - name: Set up Javascript environment uses: ./.github/actions/setup-javascript - - name: Configure sysctl limits - run: | - sudo swapoff -a - sudo sysctl -w vm.swappiness=1 - sudo sysctl -w fs.file-max=262144 - sudo sysctl -w vm.max_map_count=262144 - - - name: Install Elasticsearch - run: | - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.10-amd64.deb - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.17.10-amd64.deb.sha512 - shasum -a 512 -c elasticsearch-7.17.10-amd64.deb.sha512 - sudo dpkg -i elasticsearch-7.17.10-amd64.deb - sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install https://github.com/WorksApplications/elasticsearch-sudachi/releases/download/v3.1.0/elasticsearch-7.17.10-analysis-sudachi-3.1.0.zip - - - name: Install dictionary - run: | - wget http://sudachi.s3-website-ap-northeast-1.amazonaws.com/sudachidict/sudachi-dictionary-latest-core.zip - unzip sudachi-dictionary-latest-core.zip - sudo mkdir /etc/elasticsearch/sudachi -p - sudo cp sudachi-dictionary-*/system_core.dic /etc/elasticsearch/sudachi - - - name: Running Elasticsearch - run: | - sudo systemctl start elasticsearch - - name: Load database schema run: './bin/rails db:create db:schema:load db:seed' diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb index 0b44ab5f1677a7..9503fb85a18c2b 100644 --- a/app/chewy/accounts_index.rb +++ b/app/chewy/accounts_index.rb @@ -1,7 +1,72 @@ # frozen_string_literal: true class AccountsIndex < Chewy::Index - settings index: index_preset(refresh_interval: '30s'), analysis: { + DEVELOPMENT_SETTINGS = { + filter: { + english_stop: { + type: 'stop', + stopwords: '_english_', + }, + + english_stemmer: { + type: 'stemmer', + language: 'english', + }, + + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english', + }, + }, + + analyzer: { + natural: { + tokenizer: 'standard', + filter: %w( + lowercase + asciifolding + cjk_width + elision + english_possessive_stemmer + english_stop + english_stemmer + ), + }, + + sudachi_analyzer: { + tokenizer: 'standard', + filter: %w( + lowercase + asciifolding + cjk_width + elision + english_possessive_stemmer + english_stop + english_stemmer + ), + }, + + verbatim: { + tokenizer: 'standard', + filter: %w(lowercase asciifolding cjk_width), + }, + + edge_ngram: { + tokenizer: 'edge_ngram', + filter: %w(lowercase asciifolding cjk_width), + }, + }, + + tokenizer: { + edge_ngram: { + type: 'edge_ngram', + min_gram: 1, + max_gram: 15, + }, + }, + }.freeze + + PRODUCTION_SETTINGS = { filter: { english_stop: { type: 'stop', @@ -77,7 +142,9 @@ class AccountsIndex < Chewy::Index discard_punctuation: 'true', }, }, - } + }.freeze + + settings index: index_preset(refresh_interval: '30s'), analysis: Rails.env.test? ? DEVELOPMENT_SETTINGS : PRODUCTION_SETTINGS index_scope ::Account.searchable.includes(:account_stat) diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb index dc129434e46813..ac988d0cde5982 100644 --- a/app/chewy/public_statuses_index.rb +++ b/app/chewy/public_statuses_index.rb @@ -1,7 +1,69 @@ # frozen_string_literal: true class PublicStatusesIndex < Chewy::Index - settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: { + DEVELOPMENT_SETTINGS = { + filter: { + english_stop: { + type: 'stop', + stopwords: '_english_', + }, + + english_stemmer: { + type: 'stemmer', + language: 'english', + }, + + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english', + }, + }, + + analyzer: { + verbatim: { + tokenizer: 'uax_url_email', + filter: %w(lowercase), + }, + + content: { + tokenizer: 'standard', + filter: %w( + lowercase + asciifolding + cjk_width + elision + english_possessive_stemmer + english_stop + english_stemmer + ), + }, + + sudachi_analyzer: { + tokenizer: 'standard', + filter: %w( + lowercase + asciifolding + cjk_width + elision + english_possessive_stemmer + english_stop + english_stemmer + ), + }, + + hashtag: { + tokenizer: 'keyword', + filter: %w( + word_delimiter_graph + lowercase + asciifolding + cjk_width + ), + }, + }, + }.freeze + + PRODUCTION_SETTINGS = { filter: { english_stop: { type: 'stop', @@ -75,7 +137,9 @@ class PublicStatusesIndex < Chewy::Index discard_punctuation: 'true', }, }, - } + }.freeze + + settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: Rails.env.test? ? DEVELOPMENT_SETTINGS : PRODUCTION_SETTINGS index_scope ::Status.unscoped .kept diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index fbceb69de76de8..82ddd070050368 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -1,7 +1,68 @@ # frozen_string_literal: true class StatusesIndex < Chewy::Index - settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: { + DEVELOPMENT_SETTINGS = { + filter: { + english_stop: { + type: 'stop', + stopwords: '_english_', + }, + + english_stemmer: { + type: 'stemmer', + language: 'english', + }, + + english_possessive_stemmer: { + type: 'stemmer', + language: 'possessive_english', + }, + }, + analyzer: { + verbatim: { + tokenizer: 'uax_url_email', + filter: %w(lowercase), + }, + + content: { + tokenizer: 'standard', + filter: %w( + lowercase + asciifolding + cjk_width + elision + english_possessive_stemmer + english_stop + english_stemmer + ), + }, + + sudachi_analyzer: { + tokenizer: 'standard', + filter: %w( + lowercase + asciifolding + cjk_width + elision + english_possessive_stemmer + english_stop + english_stemmer + ), + }, + + hashtag: { + tokenizer: 'keyword', + filter: %w( + word_delimiter_graph + lowercase + asciifolding + cjk_width + ), + }, + }, + }.freeze + + PRODUCTION_SETTINGS = { filter: { english_stop: { type: 'stop', @@ -79,7 +140,9 @@ class StatusesIndex < Chewy::Index discard_punctuation: 'true', }, }, - } + }.freeze + + settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: Rails.env.test? ? DEVELOPMENT_SETTINGS : PRODUCTION_SETTINGS index_scope ::Status.unscoped.kept.without_reblogs.includes( :account,