Merge branch 'kb_migration' into kb_development

kmycode · Sep 10, 2023 · 204e6d8 · 204e6d8
2 parents 9aa4561 + 75b39e5
commit 204e6d8
Show file tree

Hide file tree

Showing 138 changed files with 2,744 additions and 995 deletions.
diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml
@@ -250,3 +250,116 @@ jobs:
         with:
           name: e2e-screenshots
           path: tmp/screenshots/
+
+  test-search:
+    name: Testing search
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+      elasticsearch:
+        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+        ports:
+          - 9200:9200
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      BUNDLE_WITH: test
+      ES_ENABLED: true
+      ES_HOST: localhost
+      ES_PORT: 9200
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '3.0'
+          - '3.1'
+          - '.ruby-version'
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Update package index
+        run: sudo apt-get update
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+          node-version-file: '.nvmrc'
+
+      - name: Install native Ruby dependencies
+        run: sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Install additional system dependencies
+        run: sudo apt-get install -y ffmpeg imagemagick
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          bundler-cache: true
+
+      - run: yarn --frozen-lockfile
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bundle exec rake spec:search
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: test-search-logs-${{ matrix.ruby-version }}
+          path: log/
+
+      - name: Archive test screenshots
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: test-search-screenshots
+          path: tmp/screenshots/
diff --git a/.nvmrc b/.nvmrc
@@ -1 +1 @@
-16.20
+20.6
diff --git a/CHANGELOG.md b/CHANGELOG.md
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.4
-# This needs to be bullseye-slim because the Ruby image is built on bullseye-slim
-ARG NODE_VERSION="16.20-bullseye-slim"
+# This needs to be bookworm-slim because the Ruby image is built on bookworm-slim
+ARG NODE_VERSION="20.6-bookworm-slim"
 
 FROM ghcr.io/moritzheiber/ruby-jemalloc:3.2.2-slim as ruby
 FROM node:${NODE_VERSION} as build
@@ -20,7 +20,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends build-essential \
         git \
         libicu-dev \
-        libidn11-dev \
+        libidn-dev \
         libpq-dev \
         libjemalloc-dev \
         zlib1g-dev \
@@ -64,13 +64,13 @@ RUN apt-get update && \
     apt-get -y --no-install-recommends install whois \
         wget \
         procps \
-        libssl1.1 \
+        libssl3 \
         libpq5 \
         imagemagick \
         ffmpeg \
         libjemalloc2 \
-        libicu67 \
-        libidn11 \
+        libicu72 \
+        libidn12 \
         libyaml-0-2 \
         file \
         ca-certificates \

diff --git a/FEDERATION.md b/FEDERATION.md
@@ -27,4 +27,5 @@ More information on HTTP Signatures, as well as examples, can be found here: htt
 
 - Linked-Data Signatures: https://docs.joinmastodon.org/spec/security/#ld
 - Bearcaps: https://docs.joinmastodon.org/spec/bearcaps/
-- Followers collection synchronization: https://git.activitypub.dev/ActivityPubDev/Fediverse-Enhancement-Proposals/src/branch/main/feps/fep-8fcf.md
+- Followers collection synchronization: https://codeberg.org/fediverse/fep/src/branch/main/fep/8fcf/fep-8fcf.md
+- Search indexing consent for actors: https://codeberg.org/fediverse/fep/src/branch/main/fep/5feb/fep-5feb.md
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -520,7 +520,7 @@ GEM
     pastel (0.8.0)
       tty-color (~> 0.5)
     pg (1.5.4)
-    pghero (3.3.3)
+    pghero (3.3.4)
       activerecord (>= 6)
     posix-spawn (0.3.15)
     premailer (1.21.0)
@@ -642,7 +642,7 @@ GEM
       sidekiq (>= 5, < 8)
     rspec-support (3.12.1)
     rspec_chunked (0.6)
-    rubocop (1.56.1)
+    rubocop (1.56.2)
       base64 (~> 0.1.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)

diff --git a/SECURITY.md b/SECURITY.md
@@ -13,9 +13,9 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 ## Supported Versions
 
-| Version | Supported |
-| ------- | --------- |
-| 4.1.x   | Yes       |
-| 4.0.x   | Yes       |
-| 3.5.x   | Yes       |
-| < 3.5   | No        |
+| Version | Supported        |
+| ------- | ---------------- |
+| 4.1.x   | Yes              |
+| 4.0.x   | Until 2023-10-31 |
+| 3.5.x   | Until 2023-12-31 |
+| < 3.5   | No               |
diff --git a/Vagrantfile b/Vagrantfile
@@ -76,7 +76,8 @@ path.logs: /var/log/elasticsearch
 network.host: 0.0.0.0
 http.port: 9200
 discovery.seed_hosts: ["localhost"]
-cluster.initial_master_nodes: ["node-1"]' > /etc/elasticsearch/elasticsearch.yml
+cluster.initial_master_nodes: ["node-1"]
+xpack.security.enabled: false' > /etc/elasticsearch/elasticsearch.yml
 
 sudo systemctl restart elasticsearch
 

diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb
@@ -37,6 +37,16 @@ class PublicStatusesIndex < Chewy::Index
           english_stemmer
         ),
       },
+
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
     },
   }.freeze
 
@@ -81,6 +91,16 @@ class PublicStatusesIndex < Chewy::Index
         ),
       },
 
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
+
       sudachi_analyzer: {
         tokenizer: 'sudachi_tokenizer',
         type: 'custom',
@@ -111,12 +131,13 @@ class PublicStatusesIndex < Chewy::Index
   index_scope ::Status.unscoped
                       .kept
                       .indexable
-                      .includes(:media_attachments, :preloadable_poll, :preview_cards)
+                      .includes(:media_attachments, :preloadable_poll, :preview_cards, :tags)
 
   root date_detection: false do
     field(:id, type: 'long')
     field(:account_id, type: 'long')
-    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'content') }
+    field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') }
+    field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
     field(:language, type: 'keyword')
     field(:domain, type: 'keyword', value: ->(status) { status.account.domain || '' })
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })

diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
@@ -36,6 +36,16 @@ class StatusesIndex < Chewy::Index
           english_stemmer
         ),
       },
+
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
     },
   }.freeze
 
@@ -84,6 +94,16 @@ class StatusesIndex < Chewy::Index
         ),
       },
 
+      hashtag: {
+        tokenizer: 'keyword',
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
+      },
+
       sudachi_analyzer: {
         tokenizer: 'sudachi_tokenizer',
         type: 'custom',
@@ -119,6 +139,7 @@ class StatusesIndex < Chewy::Index
     :local_reblogged,
     :local_bookmarked,
     :local_emoji_reacted,
+    :tags,
     :local_referenced,
     preloadable_poll: :local_voters
   ),
@@ -134,6 +155,7 @@ class StatusesIndex < Chewy::Index
     field(:id, type: 'long')
     field(:account_id, type: 'long')
     field(:text, type: 'text', analyzer: 'sudachi_analyzer', value: ->(status) { status.searchable_text }) { field(:stemmed, type: 'text', analyzer: 'sudachi_analyzer') }
+    field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
     field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
     field(:searchability, type: 'keyword', value: ->(status) { status.compute_searchability })
     field(:language, type: 'keyword')

diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
@@ -5,12 +5,21 @@ class TagsIndex < Chewy::Index
     analyzer: {
       content: {
         tokenizer: 'keyword',
-        filter: %w(lowercase asciifolding cjk_width),
+        filter: %w(
+          word_delimiter_graph
+          lowercase
+          asciifolding
+          cjk_width
+        ),
       },
 
       edge_ngram: {
         tokenizer: 'edge_ngram',
-        filter: %w(lowercase asciifolding cjk_width),
+        filter: %w(
+          lowercase
+          asciifolding
+          cjk_width
+        ),
       },
     },
 
@@ -30,12 +39,9 @@ class TagsIndex < Chewy::Index
   end
 
   root date_detection: false do
-    field :name, type: 'text', analyzer: 'content' do
-      field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
-    end
-
-    field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
-    field :usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts }
-    field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    field(:name, type: 'text', analyzer: 'content', value: :display_name) { field(:edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content') }
+    field(:reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? })
+    field(:usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts })
+    field(:last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at })
   end
 end
diff --git a/app/controllers/admin/software_updates_controller.rb b/app/controllers/admin/software_updates_controller.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Admin
+  class SoftwareUpdatesController < BaseController
+    before_action :check_enabled!
+
+    def index
+      authorize :software_update, :index?
+      @software_updates = SoftwareUpdate.all.sort_by(&:gem_version)
+    end
+
+    private
+
+    def check_enabled!
+      not_found unless SoftwareUpdate.check_enabled?
+    end
+  end
+end
diff --git a/app/controllers/api/v1/directories_controller.rb b/app/controllers/api/v1/directories_controller.rb
@@ -16,7 +16,9 @@ def require_enabled!
   end
 
   def set_accounts
-    @accounts = accounts_scope.offset(params[:offset]).limit(limit_param(DEFAULT_ACCOUNTS_LIMIT))
+    with_read_replica do
+      @accounts = accounts_scope.offset(params[:offset]).limit(limit_param(DEFAULT_ACCOUNTS_LIMIT))
+    end
   end
 
   def accounts_scope

diff --git a/app/controllers/api/v1/peers/search_controller.rb b/app/controllers/api/v1/peers/search_controller.rb
@@ -41,5 +41,7 @@ def set_domains
       domain = TagManager.instance.normalize_domain(domain)
       @domains = Instance.searchable.where(Instance.arel_table[:domain].matches("#{Instance.sanitize_sql_like(domain)}%", false, true)).limit(10).pluck(:domain)
     end
+  rescue Addressable::URI::InvalidURIError
+    @domains = []
   end
 end