Make max field mappings configurable

VedaWebProject · Jul 12, 2024 · 8eff79b · 8eff79b
1 parent 8fe8072
commit 8eff79b
Show file tree

Hide file tree

Showing 13 changed files with 57 additions and 6 deletions.
diff --git a/.env.docker b/.env.docker
@@ -76,6 +76,9 @@ TEKST_ES__HOST=es
 # TEKST_ES__INIT_TIMEOUT_S=120
 # default: 120
 
+# TEKST_ES__MAX_FIELD_MAPPINGS=1000
+# default: 1000
+
 
 # ================ SECURITY CONFIG ================
 

diff --git a/Tekst-API/.env.template b/Tekst-API/.env.template
@@ -76,6 +76,9 @@
 # TEKST_ES__INIT_TIMEOUT_S=120
 # default: 120
 
+# TEKST_ES__MAX_FIELD_MAPPINGS=1000
+# default: 1000
+
 
 # ================ SECURITY CONFIG ================
 

diff --git a/Tekst-API/openapi.json b/Tekst-API/openapi.json
@@ -11904,6 +11904,10 @@
             },
             "type": "object",
             "title": "Tekst"
+          },
+          "maxFieldMappings": {
+            "type": "integer",
+            "title": "Maxfieldmappings"
           }
         },
         "type": "object",
@@ -11913,7 +11917,8 @@
           "security",
           "systemSegments",
           "infoSegments",
-          "tekst"
+          "tekst",
+          "maxFieldMappings"
         ],
         "title": "PlatformData",
         "description": "Platform data used by the web client"

diff --git a/Tekst-API/tekst/config.py b/Tekst-API/tekst/config.py
@@ -143,6 +143,7 @@ class ElasticsearchConfig(ConfigSubSection):
     port: int = 9200
     prefix: str = "tekst"
     init_timeout_s: int = 120
+    max_field_mappings: int = 1000
 
     @field_validator("host", mode="before")
     @classmethod

diff --git a/Tekst-API/tekst/models/platform.py b/Tekst-API/tekst/models/platform.py
@@ -27,6 +27,7 @@ class PlatformData(ModelBase):
     system_segments: list[ClientSegmentRead]
     info_segments: list[ClientSegmentHead]
     tekst: dict[str, str]
+    max_field_mappings: int
 
 
 class TextStats(ModelBase):

diff --git a/Tekst-API/tekst/routers/platform.py b/Tekst-API/tekst/routers/platform.py
@@ -65,6 +65,7 @@ async def get_platform_data(ou: OptionalUserDep, cfg: ConfigDep) -> dict:
         .project(ClientSegmentHead)
         .to_list(),
         tekst=camelize(cfg.tekst),
+        max_field_mappings=cfg.es.max_field_mappings,
     )
 
 

diff --git a/Tekst-API/tekst/search/templates.py b/Tekst-API/tekst/search/templates.py
@@ -18,7 +18,12 @@
         "index": {
             "number_of_shards": 1,
             "number_of_replicas": 0,
-            "mapping": {"total_fields": {"limit": 2000}},
+            "mapping": {
+                "total_fields": {
+                    "limit": _cfg.es.max_field_mappings,
+                    "ignore_dynamic_beyond_limit": True,
+                }
+            },
         },
         "analysis": {
             "analyzer": {

diff --git a/Tekst-Web/src/api/schema.d.ts b/Tekst-Web/src/api/schema.d.ts
@@ -3088,6 +3088,8 @@ export interface components {
       tekst: {
         [key: string]: string;
       };
+      /** Maxfieldmappings */
+      maxFieldMappings: number;
     };
     /** PlatformDescriptionTranslation */
     PlatformDescriptionTranslation: {

diff --git a/Tekst-Web/src/views/admin/AdminSystemMaintenanceView.vue b/Tekst-Web/src/views/admin/AdminSystemMaintenanceView.vue
@@ -86,6 +86,17 @@ async function loadIndexInfo() {
   }
 }
 
+function getFieldMappingsStatus(fields: number) {
+  const maxFieldMappings = pfData.value?.maxFieldMappings || 0;
+  if (fields > maxFieldMappings * 0.9) {
+    return 'over';
+  } else if (fields > maxFieldMappings * 0.75) {
+    return 'near';
+  } else {
+    return 'ok';
+  }
+}
+
 onBeforeMount(() => {
   loadIndexInfo();
   updateAllTasksData();
@@ -134,13 +145,21 @@ onBeforeMount(() => {
           </tr>
         </thead>
         <template v-for="(value, key) in indexInfo" :key="key">
-          <tr v-if="!['createdAt', 'textId'].includes(key)">
+          <tr v-if="!['createdAt', 'textId', 'fields'].includes(key)">
             <th style="font-weight: var(--font-weight-normal)">
               {{ $t(`admin.system.maintenance.indices.${key}`) }}
             </th>
             <td>{{ value }}</td>
           </tr>
         </template>
+        <tr>
+          <th style="font-weight: var(--font-weight-normal)">
+            {{ $t(`admin.system.maintenance.indices.fields`) }}
+          </th>
+          <td :class="`max-fields-warn-${getFieldMappingsStatus(indexInfo.fields)}`">
+            {{ indexInfo.fields }} / {{ pfData?.maxFieldMappings || '???' }}
+          </td>
+        </tr>
         <tr>
           <th style="font-weight: var(--font-weight-normal)">
             {{ $t(`admin.system.maintenance.indices.createdAt`) }}
@@ -272,4 +291,14 @@ onBeforeMount(() => {
 .content-block :deep(.n-table td.nowrap) {
   white-space: nowrap;
 }
+
+.max-fields-warn-near {
+  color: var(--col-warning);
+  font-weight: var(--font-weight-bold);
+}
+
+.max-fields-warn-over {
+  color: var(--col-error);
+  font-weight: var(--font-weight-bold);
+}
 </style>
diff --git a/Tekst-Web/src/views/admin/AdminSystemSettingsView.vue b/Tekst-Web/src/views/admin/AdminSystemSettingsView.vue
@@ -124,7 +124,7 @@ function resetForm() {
       />
 
       <n-divider />
-      <h3>{{ $t('admin.system.platformSettings.headingOptions') }}</h3>
+      <h3>{{ $t('admin.system.platformSettings.headingConfig') }}</h3>
 
       <!-- DEFAULT TEXT -->
       <n-form-item path="defaultTextId" :label="$t('models.platformSettings.defaultText')">

diff --git a/Tekst-Web/translations/ui/deDE.yml b/Tekst-Web/translations/ui/deDE.yml
@@ -758,7 +758,7 @@ admin:
     platformSettings:
       heading: Einstellungen
       headingInfo: Plattform-Informationen
-      headingOptions: Optionen
+      headingConfig: Allgemeine Konfiguration
       msgSaved: Plattform-Einstellungen gespeichert.
       defaultTextPlaceholder: Automatisch
       formLabelDisplay: Darstellung

diff --git a/Tekst-Web/translations/ui/enUS.yml b/Tekst-Web/translations/ui/enUS.yml
@@ -733,7 +733,7 @@ admin:
     platformSettings:
       heading: Settings
       headingInfo: Platform Information
-      headingOptions: Options
+      headingConfig: General Configuration
       msgSaved: Platform settings saved.
       defaultTextPlaceholder: Automatic
       formLabelDisplay: Display

diff --git a/docs/content/setup/configuration.md b/docs/content/setup/configuration.md
@@ -44,6 +44,7 @@ Configuration for the connection to the Elasticsearch server
 | `TEKST_ES__PORT` | Elasticsearch port (Integer – default: `9200`) |
 | `TEKST_ES__PREFIX` | Elasticsearch prefix (for index, templates, etc.) (String – default: `tekst`) |
 | `TEKST_ES__INIT_TIMEOUT_S` | Timeout for waiting for Elasticsearch service to be available on startup (Integer – default: `120`) |
+| `TEKST_ES__MAX_FIELD_MAPPINGS` | Max. number of field mappings per search index – given there is enough memory, this can be increased in case there are e.g. annotation resources with many distinct annotation keys (these are dynamically mapped fields). The admin maintenance UI shows a warning if an index is about to hit this value. Any field mapping surpassing this value will be ignored and won't be searchable. (Integer – default: `1000`) |
 
 
 ## Security