[ML] Categorization wizard (elastic#53009) (elastic#54353)

* [ML] Categorization wizard

* fixing js prettier issues

* adding basic category field validation

* adding rare or count selection

* fixing types

* category examples changes

* improving results search

* adding analyzer editing

* improving callout

* updating callout text

* fixing import path

* resetting cat analyser json on flyout open

* disabling model plot by default

* minor refactoring

* fixing types

* hide estimate bucket span

* setting default bucket span

* removing ml_classic workaround

* changing style of detector selection

* fixing convert to advanced issue

* removing sparse data checkbox

* changes based on review

* use default mml

* fixing job cloning

* changes based on review

* removing categorization_analyzer from job if it is same as default

* fixing translations

* disabling model plot for rare jobs

* removing console.error in useResolver

x-pack/legacy/plugins/ml/common/constants/new_job.ts

@@ -9,16 +9,24 @@ export enum JOB_TYPE {
   MULTI_METRIC = 'multi_metric',
   POPULATION = 'population',
   ADVANCED = 'advanced',
+  CATEGORIZATION = 'categorization',
 }
 
 export enum CREATED_BY_LABEL {
   SINGLE_METRIC = 'single-metric-wizard',
   MULTI_METRIC = 'multi-metric-wizard',
   POPULATION = 'population-wizard',
+  CATEGORIZATION = 'categorization-wizard',
 }
 
 export const DEFAULT_MODEL_MEMORY_LIMIT = '10MB';
 export const DEFAULT_BUCKET_SPAN = '15m';
+export const DEFAULT_RARE_BUCKET_SPAN = '1h';
 export const DEFAULT_QUERY_DELAY = '60s';
 
 export const SHARED_RESULTS_INDEX_NAME = 'shared';
+
+export const NUMBER_OF_CATEGORY_EXAMPLES = 5;
+export const CATEGORY_EXAMPLES_MULTIPLIER = 20;
+export const CATEGORY_EXAMPLES_WARNING_LIMIT = 0.75;
+export const CATEGORY_EXAMPLES_ERROR_LIMIT = 0.2;

x-pack/legacy/plugins/ml/common/types/categories.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export type CategoryId = number;
+
+export interface Category {
+  job_id: string;
+  category_id: CategoryId;
+  terms: string;
+  regex: string;
+  max_matching_length: number;
+  examples: string[];
+  grok_pattern: string;
+}
+
+export interface Token {
+  token: string;
+  start_offset: number;
+  end_offset: number;
+  type: string;
+  position: number;
+}

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts

@@ -0,0 +1,159 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { isEqual } from 'lodash';
+import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
+import { SavedSearchSavedObject } from '../../../../../../common/types/kibana';
+import { JobCreator } from './job_creator';
+import { Field, Aggregation, mlCategory } from '../../../../../../common/types/fields';
+import { Job, Datafeed, Detector } from './configs';
+import { createBasicDetector } from './util/default_configs';
+import {
+  JOB_TYPE,
+  CREATED_BY_LABEL,
+  DEFAULT_BUCKET_SPAN,
+  DEFAULT_RARE_BUCKET_SPAN,
+} from '../../../../../../common/constants/new_job';
+import { ML_JOB_AGGREGATION } from '../../../../../../common/constants/aggregation_types';
+import { getRichDetectors } from './util/general';
+import { CategorizationExamplesLoader, CategoryExample } from '../results_loader';
+import { CategorizationAnalyzer, getNewJobDefaults } from '../../../../services/ml_server_info';
+
+type CategorizationAnalyzerType = CategorizationAnalyzer | null;
+
+export class CategorizationJobCreator extends JobCreator {
+  protected _type: JOB_TYPE = JOB_TYPE.CATEGORIZATION;
+  private _createCountDetector: () => void = () => {};
+  private _createRareDetector: () => void = () => {};
+  private _examplesLoader: CategorizationExamplesLoader;
+  private _categoryFieldExamples: CategoryExample[] = [];
+  private _categoryFieldValid: number = 0;
+  private _detectorType: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE =
+    ML_JOB_AGGREGATION.COUNT;
+  private _categorizationAnalyzer: CategorizationAnalyzerType = null;
+  private _defaultCategorizationAnalyzer: CategorizationAnalyzerType;
+
+  constructor(
+    indexPattern: IndexPattern,
+    savedSearch: SavedSearchSavedObject | null,
+    query: object
+  ) {
+    super(indexPattern, savedSearch, query);
+    this.createdBy = CREATED_BY_LABEL.CATEGORIZATION;
+    this._examplesLoader = new CategorizationExamplesLoader(this, indexPattern, query);
+
+    const { anomaly_detectors: anomalyDetectors } = getNewJobDefaults();
+    this._defaultCategorizationAnalyzer = anomalyDetectors.categorization_analyzer || null;
+  }
+
+  public setDefaultDetectorProperties(
+    count: Aggregation | null,
+    rare: Aggregation | null,
+    eventRate: Field | null
+  ) {
+    if (count === null || rare === null || eventRate === null) {
+      return;
+    }
+
+    this._createCountDetector = () => {
+      this._createDetector(count, eventRate);
+    };
+    this._createRareDetector = () => {
+      this._createDetector(rare, eventRate);
+    };
+  }
+
+  private _createDetector(agg: Aggregation, field: Field) {
+    const dtr: Detector = createBasicDetector(agg, field);
+    dtr.by_field_name = mlCategory.id;
+    this._addDetector(dtr, agg, mlCategory);
+  }
+
+  public setDetectorType(type: ML_JOB_AGGREGATION.COUNT | ML_JOB_AGGREGATION.RARE) {
+    this._detectorType = type;
+    this.removeAllDetectors();
+    if (type === ML_JOB_AGGREGATION.COUNT) {
+      this._createCountDetector();
+      this.bucketSpan = DEFAULT_BUCKET_SPAN;
+    } else {
+      this._createRareDetector();
+      this.bucketSpan = DEFAULT_RARE_BUCKET_SPAN;
+      this.modelPlot = false;
+    }
+  }
+
+  public set categorizationFieldName(fieldName: string | null) {
+    if (fieldName !== null) {
+      this._job_config.analysis_config.categorization_field_name = fieldName;
+      this.setDetectorType(this._detectorType);
+      this.addInfluencer(mlCategory.id);
+    } else {
+      delete this._job_config.analysis_config.categorization_field_name;
+      this._categoryFieldExamples = [];
+      this._categoryFieldValid = 0;
+    }
+  }
+
+  public get categorizationFieldName(): string | null {
+    return this._job_config.analysis_config.categorization_field_name || null;
+  }
+
+  public async loadCategorizationFieldExamples() {
+    const { valid, examples } = await this._examplesLoader.loadExamples();
+    this._categoryFieldExamples = examples;
+    this._categoryFieldValid = valid;
+    return { valid, examples };
+  }
+
+  public get categoryFieldExamples() {
+    return this._categoryFieldExamples;
+  }
+
+  public get categoryFieldValid() {
+    return this._categoryFieldValid;
+  }
+
+  public get selectedDetectorType() {
+    return this._detectorType;
+  }
+
+  public set categorizationAnalyzer(analyzer: CategorizationAnalyzerType) {
+    this._categorizationAnalyzer = analyzer;
+
+    if (
+      analyzer === null ||
+      isEqual(this._categorizationAnalyzer, this._defaultCategorizationAnalyzer)
+    ) {
+      delete this._job_config.analysis_config.categorization_analyzer;
+    } else {
+      this._job_config.analysis_config.categorization_analyzer = analyzer;
+    }
+  }
+
+  public get categorizationAnalyzer() {
+    return this._categorizationAnalyzer;
+  }
+
+  public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
+    this._overrideConfigs(job, datafeed);
+    this.createdBy = CREATED_BY_LABEL.CATEGORIZATION;
+    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
+
+    const dtr = detectors[0];
+    if (detectors.length && dtr.agg !== null && dtr.field !== null) {
+      this._detectorType =
+        dtr.agg.id === ML_JOB_AGGREGATION.COUNT
+          ? ML_JOB_AGGREGATION.COUNT
+          : ML_JOB_AGGREGATION.RARE;
+
+      const bs = job.analysis_config.bucket_span;
+      this.setDetectorType(this._detectorType);
+      // set the bucketspan back to the original value
+      // as setDetectorType applies a default
+      this.bucketSpan = bs;
+    }
+  }
+}

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/index.ts

@@ -9,11 +9,13 @@ export { SingleMetricJobCreator } from './single_metric_job_creator';
 export { MultiMetricJobCreator } from './multi_metric_job_creator';
 export { PopulationJobCreator } from './population_job_creator';
 export { AdvancedJobCreator } from './advanced_job_creator';
+export { CategorizationJobCreator } from './categorization_job_creator';
 export {
   JobCreatorType,
   isSingleMetricJobCreator,
   isMultiMetricJobCreator,
   isPopulationJobCreator,
   isAdvancedJobCreator,
+  isCategorizationJobCreator,
 } from './type_guards';
 export { jobCreatorFactory } from './job_creator_factory';

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator_factory.ts

@@ -10,6 +10,7 @@ import { MultiMetricJobCreator } from './multi_metric_job_creator';
 import { PopulationJobCreator } from './population_job_creator';
 import { AdvancedJobCreator } from './advanced_job_creator';
 import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public';
+import { CategorizationJobCreator } from './categorization_job_creator';
 
 import { JOB_TYPE } from '../../../../../../common/constants/new_job';
 
@@ -32,6 +33,9 @@ export const jobCreatorFactory = (jobType: JOB_TYPE) => (
     case JOB_TYPE.ADVANCED:
       jc = AdvancedJobCreator;
       break;
+    case JOB_TYPE.CATEGORIZATION:
+      jc = CategorizationJobCreator;
+      break;
     default:
       jc = SingleMetricJobCreator;
       break;

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/type_guards.ts

@@ -8,13 +8,15 @@ import { SingleMetricJobCreator } from './single_metric_job_creator';
 import { MultiMetricJobCreator } from './multi_metric_job_creator';
 import { PopulationJobCreator } from './population_job_creator';
 import { AdvancedJobCreator } from './advanced_job_creator';
+import { CategorizationJobCreator } from './categorization_job_creator';
 import { JOB_TYPE } from '../../../../../../common/constants/new_job';
 
 export type JobCreatorType =
   | SingleMetricJobCreator
   | MultiMetricJobCreator
   | PopulationJobCreator
-  | AdvancedJobCreator;
+  | AdvancedJobCreator
+  | CategorizationJobCreator;
 
 export function isSingleMetricJobCreator(
   jobCreator: JobCreatorType
@@ -37,3 +39,9 @@ export function isPopulationJobCreator(
 export function isAdvancedJobCreator(jobCreator: JobCreatorType): jobCreator is AdvancedJobCreator {
   return jobCreator.type === JOB_TYPE.ADVANCED;
 }
+
+export function isCategorizationJobCreator(
+  jobCreator: JobCreatorType
+): jobCreator is CategorizationJobCreator {
+  return jobCreator.type === JOB_TYPE.CATEGORIZATION;
+}

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts

@@ -19,7 +19,12 @@ import {
   mlCategory,
 } from '../../../../../../../common/types/fields';
 import { mlJobService } from '../../../../../services/job_service';
-import { JobCreatorType, isMultiMetricJobCreator, isPopulationJobCreator } from '../index';
+import {
+  JobCreatorType,
+  isMultiMetricJobCreator,
+  isPopulationJobCreator,
+  isCategorizationJobCreator,
+} from '../index';
 import { CREATED_BY_LABEL, JOB_TYPE } from '../../../../../../../common/constants/new_job';
 
 const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => {
@@ -251,6 +256,8 @@ export function convertToAdvancedJob(jobCreator: JobCreatorType) {
     jobType = JOB_TYPE.MULTI_METRIC;
   } else if (isPopulationJobCreator(jobCreator)) {
     jobType = JOB_TYPE.POPULATION;
+  } else if (isCategorizationJobCreator(jobCreator)) {
+    jobType = JOB_TYPE.CATEGORIZATION;
   }
 
   window.location.href = window.location.href.replace(jobType, JOB_TYPE.ADVANCED);

x-pack/legacy/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts

@@ -12,10 +12,11 @@ import {
   basicDatafeedValidation,
 } from '../../../../../../common/util/job_utils';
 import { getNewJobLimits } from '../../../../services/ml_server_info';
-import { JobCreator, JobCreatorType } from '../job_creator';
+import { JobCreator, JobCreatorType, isCategorizationJobCreator } from '../job_creator';
 import { populateValidationMessages, checkForExistingJobAndGroupIds } from './util';
 import { ExistingJobsAndGroups } from '../../../../services/job_service';
 import { cardinalityValidator, CardinalityValidatorResult } from './validators';
+import { CATEGORY_EXAMPLES_ERROR_LIMIT } from '../../../../../../common/constants/new_job';
 
 // delay start of validation to allow the user to make changes
 // e.g. if they are typing in a new value, try not to validate
@@ -51,6 +52,10 @@ export interface BasicValidations {
   scrollSize: Validation;
 }
 
+export interface AdvancedValidations {
+  categorizationFieldValid: Validation;
+}
+
 export class JobValidator {
   private _jobCreator: JobCreatorType;
   private _validationSummary: ValidationSummary;
@@ -71,6 +76,9 @@ export class JobValidator {
     frequency: { valid: true },
     scrollSize: { valid: true },
   };
+  private _advancedValidations: AdvancedValidations = {
+    categorizationFieldValid: { valid: true },
+  };
   private _validating: boolean = false;
   private _basicValidationResult$ = new ReplaySubject<JobValidationResult>(2);
 
@@ -141,6 +149,7 @@ export class JobValidator {
       this._lastDatafeedConfig = formattedDatafeedConfig;
       this._validateTimeout = setTimeout(() => {
         this._runBasicValidation();
+        this._runAdvancedValidation();
 
         this._jobCreatorSubject$.next(this._jobCreator);
 
@@ -195,6 +204,13 @@ export class JobValidator {
     this._basicValidationResult$.next(this._basicValidations);
   }
 
+  private _runAdvancedValidation() {
+    if (isCategorizationJobCreator(this._jobCreator)) {
+      this._advancedValidations.categorizationFieldValid.valid =
+        this._jobCreator.categoryFieldValid > CATEGORY_EXAMPLES_ERROR_LIMIT;
+    }
+  }
+
   private _isOverallBasicValid() {
     return Object.values(this._basicValidations).some(v => v.valid === false) === false;
   }
@@ -246,4 +262,12 @@ export class JobValidator {
   public get validating(): boolean {
     return this._validating;
   }
+
+  public get categorizationField() {
+    return this._advancedValidations.categorizationFieldValid.valid;
+  }
+
+  public set categorizationField(valid: boolean) {
+    this._advancedValidations.categorizationFieldValid.valid = valid;
+  }
 }