diff --git a/tests/cypress/e2e/features/ground_truth_jobs.js b/tests/cypress/e2e/features/ground_truth_jobs.js index 0753d59839cc..01cac1da5f6f 100644 --- a/tests/cypress/e2e/features/ground_truth_jobs.js +++ b/tests/cypress/e2e/features/ground_truth_jobs.js @@ -4,12 +4,11 @@ /// +import { defaultTaskSpec } from '../../support/default-specs'; + context('Ground truth jobs', () => { - const caseId = 'Ground truth jobs'; const labelName = 'car'; - const taskName = `Annotation task for Case ${caseId}`; - const attrName = `Attr for Case ${caseId}`; - const textDefaultValue = 'Some default value for type Text'; + const taskName = 'Annotation task for Ground truth jobs'; const jobOptions = { jobType: 'Ground truth', @@ -17,6 +16,12 @@ context('Ground truth jobs', () => { fromTaskPage: true, }; + const defaultValidationParams = { + frameCount: 3, + mode: 'gt', + frameSelectionMethod: 'random_uniform', + }; + const groundTruthRectangles = [ { id: 1, @@ -64,8 +69,8 @@ context('Ground truth jobs', () => { let jobID = null; let taskID = null; - // With seed = 1, frameCount = 4, totalFrames = 10 - predifined ground truth frames are: - const groundTruthFrames = [0, 1, 5, 6]; + // With seed = 1, frameCount = 4, totalFrames = 100 - predifined ground truth frames are: + const groundTruthFrames = [10, 23, 71, 87]; function checkRectangleAndObjectMenu(rectangle, isGroundTruthJob = false) { if (isGroundTruthJob) { @@ -97,36 +102,33 @@ context('Ground truth jobs', () => { cy.get('.cvat-quality-control-management-tab').should('exist').and('be.visible'); } + function createAndOpenTask(serverFiles, validationParams = null) { + const { taskSpec, dataSpec, extras } = defaultTaskSpec({ + taskName, serverFiles, labelName, validationParams, + }); + return cy.headlessCreateTask(taskSpec, dataSpec, extras).then((taskResponse) => { + taskID = taskResponse.taskID; + if (validationParams) { + [groundTruthJobID, jobID] = taskResponse.jobIDs; + } else { + [jobID] = taskResponse.jobIDs; + } + }).then(() => { + cy.visit(`/tasks/${taskID}`); + cy.get('.cvat-task-details').should('exist').and('be.visible'); + }); + } + before(() => { cy.visit('auth/login'); cy.login(); }); describe('Testing ground truth basics', () => { - const imagesCount = 10; - const imageFileName = 'ground_truth_1'; - const width = 800; - const height = 800; - const posX = 10; - const posY = 10; - const color = 'gray'; - const archiveName = `${imageFileName}.zip`; - const archivePath = `cypress/fixtures/${archiveName}`; - const imagesFolder = `cypress/fixtures/${imageFileName}`; - const directoryToArchive = imagesFolder; + const serverFiles = ['bigArchive.zip']; before(() => { - cy.visit('/tasks'); - cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); - cy.createZipArchive(directoryToArchive, archivePath); - cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, archiveName); - cy.openTask(taskName); - cy.url().then((url) => { - taskID = Number(url.split('/').slice(-1)[0].split('?')[0]); - }); - cy.get('.cvat-job-item').first().invoke('attr', 'data-row-id').then((val) => { - jobID = val; - }); + createAndOpenTask(serverFiles); }); after(() => { @@ -196,35 +198,80 @@ context('Ground truth jobs', () => { }); }); + describe('Testing creating task with quality params', () => { + const imagesCount = 3; + const imageFileName = `image_${taskName.replace(' ', '_').toLowerCase()}`; + const width = 800; + const height = 800; + const posX = 10; + const posY = 10; + const color = 'gray'; + const archiveName = `${imageFileName}.zip`; + const archivePath = `cypress/fixtures/${archiveName}`; + const imagesFolder = `cypress/fixtures/${imageFileName}`; + const directoryToArchive = imagesFolder; + const attrName = 'gt_attr'; + const defaultAttrValue = 'GT attr'; + const multiAttrParams = false; + const forProject = false; + const attachToProject = false; + const projectName = null; + const expectedResult = 'success'; + const projectSubsetFieldValue = null; + const advancedConfigurationParams = false; + + before(() => { + cy.contains('.cvat-header-button', 'Tasks').should('be.visible').click(); + cy.url().should('include', '/tasks'); + cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); + cy.createZipArchive(directoryToArchive, archivePath); + }); + + afterEach(() => { + cy.goToTaskList(); + cy.deleteTask(taskName); + }); + + function createTaskWithQualityParams(qualityParams) { + cy.createAnnotationTask( + taskName, + labelName, + attrName, + defaultAttrValue, + archiveName, + multiAttrParams, + advancedConfigurationParams, + forProject, + attachToProject, + projectName, + expectedResult, + projectSubsetFieldValue, + qualityParams, + ); + cy.openTask(taskName); + cy.get('.cvat-job-item').first() + .find('.ant-tag') + .should('have.text', 'Ground truth'); + } + + it('Create task with ground truth job', () => { + createTaskWithQualityParams({ + validationMode: 'Ground Truth', + }); + }); + + it('Create task with honeypots', () => { + createTaskWithQualityParams({ + validationMode: 'Honeypots', + }); + }); + }); + describe('Testing ground truth management basics', () => { const serverFiles = ['images/image_1.jpg', 'images/image_2.jpg', 'images/image_3.jpg']; before(() => { - cy.headlessCreateTask({ - labels: [{ name: labelName, attributes: [], type: 'any' }], - name: taskName, - project_id: null, - source_storage: { location: 'local' }, - target_storage: { location: 'local' }, - }, { - server_files: serverFiles, - image_quality: 70, - use_zip_chunks: true, - use_cache: true, - sorting_method: 'lexicographical', - }).then((taskResponse) => { - taskID = taskResponse.taskID; - [jobID] = taskResponse.jobIDs; - }).then(() => ( - cy.headlessCreateJob({ - task_id: taskID, - frame_count: 3, - type: 'ground_truth', - frame_selection_method: 'random_uniform', - }) - )).then((jobResponse) => { - groundTruthJobID = jobResponse.jobID; - }).then(() => { + createAndOpenTask(serverFiles, defaultValidationParams).then(() => { cy.visit(`/tasks/${taskID}/quality-control#management`); cy.get('.cvat-quality-control-management-tab').should('exist').and('be.visible'); cy.get('.cvat-annotations-quality-allocation-table-summary').should('exist').and('be.visible'); @@ -312,35 +359,10 @@ context('Ground truth jobs', () => { }); describe('Regression tests', () => { - const imagesCount = 20; - const imageFileName = 'ground_truth_2'; - const width = 100; - const height = 100; - const posX = 10; - const posY = 10; - const color = 'gray'; - const archiveName = `${imageFileName}.zip`; - const archivePath = `cypress/fixtures/${archiveName}`; - const imagesFolder = `cypress/fixtures/${imageFileName}`; - const directoryToArchive = imagesFolder; + const serverFiles = ['bigArchive.zip']; before(() => { - cy.visit('/tasks'); - cy.imageGenerator(imagesFolder, imageFileName, width, height, color, posX, posY, labelName, imagesCount); - cy.createZipArchive(directoryToArchive, archivePath); - cy.createAnnotationTask( - taskName, - labelName, - attrName, - textDefaultValue, - archiveName, - false, - { multiJobs: true, segmentSize: 1 }, - ); - cy.openTask(taskName); - cy.url().then((url) => { - taskID = Number(url.split('/').slice(-1)[0].split('?')[0]); - }); + createAndOpenTask(serverFiles); }); afterEach(() => { diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index d3988c2e56a0..f0f085260cbf 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -178,6 +178,7 @@ Cypress.Commands.add( projectName = '', expectedResult = 'success', projectSubsetFieldValue = 'Test', + qualityConfigurationParams = null, ) => { cy.url().then(() => { cy.get('.cvat-create-task-dropdown').click(); @@ -215,6 +216,9 @@ Cypress.Commands.add( if (advancedConfigurationParams) { cy.advancedConfiguration(advancedConfigurationParams); } + if (qualityConfigurationParams) { + cy.configureTaskQualityMode(qualityConfigurationParams); + } cy.get('.cvat-submit-continue-task-button').scrollIntoView(); cy.get('.cvat-submit-continue-task-button').click(); if (expectedResult === 'success') { @@ -291,7 +295,7 @@ Cypress.Commands.add('headlessCreateObjects', (objects, jobID) => { }); }); -Cypress.Commands.add('headlessCreateTask', (taskSpec, dataSpec) => { +Cypress.Commands.add('headlessCreateTask', (taskSpec, dataSpec, extras) => { cy.window().then(async ($win) => { const task = new $win.cvat.classes.Task({ ...taskSpec, @@ -310,7 +314,7 @@ Cypress.Commands.add('headlessCreateTask', (taskSpec, dataSpec) => { task.remoteFiles = dataSpec.remote_files; } - const result = await task.save(); + const result = await task.save(extras || {}); return cy.wrap({ taskID: result.id, jobIDs: result.jobs.map((job) => job.id) }); }); }); @@ -897,6 +901,15 @@ Cypress.Commands.add('advancedConfiguration', (advancedConfigurationParams) => { } }); +Cypress.Commands.add('configureTaskQualityMode', (qualityConfigurationParams) => { + cy.contains('Quality').click(); + if (qualityConfigurationParams.validationMode) { + cy.get('#validationMode').within(() => { + cy.contains(qualityConfigurationParams.validationMode).click(); + }); + } +}); + Cypress.Commands.add('removeAnnotations', () => { cy.contains('.cvat-annotation-header-button', 'Menu').click(); cy.get('.cvat-annotation-menu').within(() => { diff --git a/tests/cypress/support/default-specs.js b/tests/cypress/support/default-specs.js new file mode 100644 index 000000000000..ea07bab747b2 --- /dev/null +++ b/tests/cypress/support/default-specs.js @@ -0,0 +1,63 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +function defaultTaskSpec({ + labelName, + taskName, + serverFiles, + validationParams, +}) { + const taskSpec = { + labels: [ + { name: labelName, attributes: [], type: 'any' }, + ], + name: taskName, + project_id: null, + source_storage: { location: 'local' }, + target_storage: { location: 'local' }, + }; + + const dataSpec = { + server_files: serverFiles, + image_quality: 70, + use_zip_chunks: true, + use_cache: true, + sorting_method: (validationParams && validationParams.mode === 'gt_pool') ? 'random' : 'lexicographical', + }; + + const extras = {}; + if (validationParams) { + const convertedParams = {}; + if (validationParams.frames) { + convertedParams.frames = validationParams.frames; + } + if (validationParams.frameSelectionMethod) { + convertedParams.frame_selection_method = validationParams.frameSelectionMethod; + } + if (validationParams.frameCount) { + convertedParams.frame_count = validationParams.frameCount; + } + if (validationParams.framesPerJobCount) { + convertedParams.frames_per_job_count = validationParams.framesPerJobCount; + } + if (validationParams.mode) { + convertedParams.mode = validationParams.mode; + } + if (validationParams.randomSeed) { + convertedParams.random_seed = validationParams.randomSeed; + } + + extras.validation_params = convertedParams; + } + + return { + taskSpec, + dataSpec, + extras, + }; +} + +module.exports = { + defaultTaskSpec, +};