From d20ed4e329aaefe76d952f3a71090e2bb5921842 Mon Sep 17 00:00:00 2001 From: Christian Lanig Date: Fri, 12 May 2023 15:22:44 +0200 Subject: [PATCH 1/4] Display OCR result of test on step screenshot --- assets/javascripts/needlediff.js | 82 ++++++++++++++++++++++++---- lib/OpenQA/WebAPI/Controller/Step.pm | 49 ++++++++++++++++- 2 files changed, 117 insertions(+), 14 deletions(-) diff --git a/assets/javascripts/needlediff.js b/assets/javascripts/needlediff.js index 737ba0a6da9..b6ed9431b72 100644 --- a/assets/javascripts/needlediff.js +++ b/assets/javascripts/needlediff.js @@ -77,6 +77,64 @@ function NeedleDiff(id, width, height) { }); } +/** + * Function ensuring that the text fits into the area rectangle. + * @param {Object} ctx - The Canvas context. + * @param {Object} area - The OCR area to be drawn into. + * @param {Object} txtStyle - The target text style for calibration. + * @returns {number} The calculated font size. + */ +function calcFontSize(ctx, area, txtStyle) { + const ocrLines = area.ocr_str.split('\n'); + + // 1px margin in height + let ret = Math.ceil(((area.height - ocrLines.length - 1) / ocrLines.length) - 1); + + ctx.font = txtStyle.weight + ' ' + ret + 'px ' + txtStyle.typeface; + for (line of ocrLines) { + let ocrLinesWidth = ctx.measureText(line).width; + while (ocrLinesWidth > area.width - 2) { + ret--; + ctx.font = txtStyle.weight + ' ' + ret + 'px ' + txtStyle.typeface; + ocrLinesWidth = ctx.measureText(line).width; + } + } + return ret; +} + +/** + * Method drawing the OCR text into the OCR area. + * @param {Object} needlediff - The needlediff containing the canvas context. + * @param {Object} area - The OCR area to be drawn into. + */ +function drawOcrTxt(needlediff, area) { + const txtStyle = { + alignment: 'center', + typeface: 'Arial', + color: 'rgb(64,224,208)', + weight: 'bold' + } + const fontSize = calcFontSize(needlediff.ctx, area, txtStyle); + const ocrLines = area.ocr_str.split('\n'); + const ocrTxtXPos = Math.ceil(area.xpos + area.width / 2); + + // Body line appears to be at about 1/3 of whole font height + let ocrTxtYPos = Math.floor(area.ypos + area.height / 2 + fontSize / 3 - + ((ocrLines.length - 1) * fontSize) / 2); + + needlediff.ctx.textAlign = txtStyle.alignment; + needlediff.ctx.fillStyle = txtStyle.color; + needlediff.ctx.font = txtStyle.weight + ' ' + fontSize + 'px ' + + txtStyle.typeface; + + for (line of ocrLines) { + needlediff.ctx.fillText(line, ocrTxtXPos, ocrTxtYPos); + + // 1px space between lines + ocrTxtYPos += fontSize + 1; + } +} + NeedleDiff.prototype.draw = function () { // First of all, draw the screenshot as gray background (if ready) if (!this.screenshotImg) { @@ -89,11 +147,6 @@ NeedleDiff.prototype.draw = function () { this.ctx.drawImage(this.screenshotImg, 0, 0); } - // Then, check if there is a needle to compare with - if (!this.needleImg) { - return; - } - // Calculate the pixel in which the division will be done var split = this.divide * this.width; if (split < 1) { @@ -138,7 +191,12 @@ NeedleDiff.prototype.draw = function () { if (!this.fullNeedleImg) { // draw matching part of needle image this.ctx.strokeStyle = NeedleDiff.strokecolor(a.type); - this.ctx.drawImage(this.needleImg, orig.xpos, orig.ypos, usedWith, a.height, x, a.ypos, usedWith, a.height); + if (this.needleImg) { + this.ctx.drawImage(this.needleImg, orig.xpos, orig.ypos, usedWith, a.height, x, a.ypos, usedWith, a.height); + } else if (a.ocr_str) { + drawOcrTxt(this, a); + } else return; + // draw frame of match area this.ctx.lineWidth = lineWidth; this.ctx.beginPath(); @@ -221,6 +279,7 @@ NeedleDiff.prototype.draw = function () { this.ctx.strokeStyle = 'rgb(0, 0, 0)'; this.ctx.lineWidth = 3; this.ctx.font = 'bold 14px Arial'; + this.ctx.textAlign = 'left'; var text = a['similarity'] + '%'; var textSize = this.ctx.measureText(text); var tx; @@ -326,16 +385,17 @@ function setDiffScreenshot(screenshotSrc) { $('').on('load', function () { var image = $(this).get(0); - // set screenshot resolution - window.differ = new NeedleDiff('needle_diff', image.width, image.height); - window.differ.screenshotImg = image; - setNeedle(); - // create gray version of it in off screen canvas var gray_canvas = document.createElement('canvas'); gray_canvas.width = image.width; gray_canvas.height = image.height; + // set screenshot resolution + window.differ = new NeedleDiff('needle_diff', image.width, image.height); + window.differ.screenshotImg = image; + window.differ.gray_canvas = gray_canvas; + setNeedle(); + var gray_context = gray_canvas.getContext('2d'); gray_context.drawImage(image, 0, 0); diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm index 21c11b6c067..e4ae52d9616 100644 --- a/lib/OpenQA/WebAPI/Controller/Step.pm +++ b/lib/OpenQA/WebAPI/Controller/Step.pm @@ -263,8 +263,9 @@ sub _new_screenshot ($self, $tags, $image_name, $matches = undef) { ypos => int $area->{y}, width => int $area->{w}, height => int $area->{h}, - type => 'match', ); + $match{type} = defined $area->{refstr} ? 'ocr' : 'match'; + $match{refstr} = $area->{refstr} if defined $area->{refstr}; if (my $click_point = $area->{click_point}) { $match{click_point} = $click_point; } @@ -449,6 +450,7 @@ sub calc_matches ($needle, $areas) { type => $area->{result}, similarity => int($area->{similarity} + 0.5), ); + $match{ocr_str} = $area->{ocr_str} if defined $area->{ocr_str}; if (my $click_point = $area->{click_point}) { $match{click_point} = $click_point; } @@ -458,6 +460,45 @@ sub calc_matches ($needle, $areas) { return; } +=head2 has_image + +Checks if a needle has a needle image by it's type of areas. + +=head3 Params + +=over + +=item * + +areas of object type ARRAY containing HASH objects containing at least: + +=over + +=item * + +type => s in {'ocr', 'match', 'exclude'} | type is the area type. + +=back + +=back + +=head3 returns + +i in {1, 0} | i is 1 if the area has an image. + +=cut + +sub has_image ($areas) { + my $ocr_area; + my $img_matching_area; + + for my $area (@$areas) { + $ocr_area = 1 if $area->{type} eq 'ocr'; + $img_matching_area = 1 if $area->{type} eq 'match'; + } + return $ocr_area ? $img_matching_area : 1; +} + sub viewimg ($self) { my $module_detail = $self->stash('module_detail'); my $job = $self->stash('job'); @@ -505,13 +546,14 @@ sub viewimg ($self) { my $info = { name => $needle, needledir => $needleinfo->{needledir}, - image => $self->needle_url($distri, $needle . '.png', $dversion, $needleinfo->{json}), areas => $needleinfo->{area}, error => $module_detail->{error}, matches => [], primary_match => 1, selected => 1, }; + $info->{image} = $self->needle_url($distri, $needle . '.png', $dversion, $needleinfo->{json}) + if has_image($needleinfo->{area}); calc_matches($info, $module_detail->{area}); $primary_match = $info; $append_needle_info->($needleinfo->{tags} => $info); @@ -527,11 +569,12 @@ sub viewimg ($self) { my $info = { name => $needlename, needledir => $needleinfo->{needledir}, - image => $self->needle_url($distri, "$needlename.png", $dversion, $needleinfo->{json}), error => $needle->{error}, areas => $needleinfo->{area}, matches => [], }; + $info->{image} = $self->needle_url($distri, "$needlename.png", $dversion, $needleinfo->{json}) + if has_image($needleinfo->{area}); calc_matches($info, $needle->{area}); $append_needle_info->($needleinfo->{tags} => $info); } From 882d8b1591eafe0db4b111a5c4cd2cabfeb1259c Mon Sep 17 00:00:00 2001 From: Christian Lanig Date: Tue, 12 Sep 2023 07:41:06 +0200 Subject: [PATCH 2/4] Implement OCR in needle editor --- assets/javascripts/needleeditor.js | 77 ++++++++++++++++++++++++++-- lib/OpenQA/Task/Needle/Save.pm | 55 ++++++++++++-------- lib/OpenQA/WebAPI/Controller/Step.pm | 22 +++++--- templates/webapi/step/edit.html.ep | 5 ++ 4 files changed, 125 insertions(+), 34 deletions(-) diff --git a/assets/javascripts/needleeditor.js b/assets/javascripts/needleeditor.js index ab0f04894b1..524b526bfb3 100644 --- a/assets/javascripts/needleeditor.js +++ b/assets/javascripts/needleeditor.js @@ -59,6 +59,13 @@ NeedleEditor.prototype.init = function () { return; } a.type = NeedleEditor.nexttype(a.type); + + updateAreaCtrls(a); + + // Attrs only apply to some area types and are not retained elsewhere. + delete a.refstr; + delete a.margin; + shape.fill = NeedleEditor.areacolor(a.type); editor.UpdateTextArea(); cv.redraw(); @@ -110,15 +117,14 @@ NeedleEditor.prototype.init = function () { cv.addShape(shape); editor.needle.area.push(a); editor.UpdateTextArea(); + updateAreaCtrls(a); return shape; }; - var areaSpecificButtons = $('#change-match, #change-margin, #toggle-click-coordinates'); $(cv).on('shape.selected', function () { - areaSpecificButtons.removeClass('disabled').removeAttr('disabled'); - updateToggleClickCoordinatesButton(editor.currentClickCoordinates()); + updateAreaCtrls(editor.selection().area); }); $(cv).on('shape.unselected', function () { - areaSpecificButtons.addClass('disabled').attr('disabled', 1); + updateAreaCtrls(null); }); document.getElementById('needleeditor_name').onchange = function () { @@ -160,6 +166,15 @@ NeedleEditor.prototype.AddTag = function (tag, checked) { return input; }; +/** + * Method updating the reference string and it's in it's text box and the + * corresponding attribute of it's area object. + */ +NeedleEditor.prototype.updateRefstr = function () { + this.selection().area.refstr = document.getElementById('txtarea-refstr').value; + this.UpdateTextArea(); +}; + NeedleEditor.nexttype = function (type) { if (type == 'match') { return 'exclude'; @@ -377,6 +392,7 @@ NeedleEditor.prototype.toggleClickCoordinates = function () { function loadBackground() { var needle = window.needles[$('#image_select option:selected').val()]; + needle = needle.imageurl ? needle : window.needles['screenshot']; nEditor.LoadBackground(needle.imageurl); $('#needleeditor_image').val(needle.imagename); $('#needleeditor_imagedistri').val(needle.imagedistri); @@ -440,6 +456,55 @@ function loadAreas() { } } +/** + * Method disabling a HTML element. + * @param {string} elemId - The ID of the element in the HTML document. + */ +function disableElem(elemId) { + const elem = document.getElementById(elemId); + elem.classList.add('disabled'); + elem.disabled = 'disabled'; +} + +/** + * Method enabling a HTML element. + * @param {string} elemId - The ID of the element in the HTML document. + */ +function enableElem(elemId) { + const elem = document.getElementById(elemId); + elem.classList.remove('disabled'); + elem.removeAttribute('disabled'); +} + +/** + * Method enabling/disabling controls for a specific area. + * @param {Object} area - The area which the controls should be prepared for. + */ +function updateAreaCtrls(area) { + + const ctrlElemIds = { + all: ['change-match', 'change-margin', 'toggle-click-coordinates', 'txtarea-refstr'], + match: ['change-match', 'change-margin', 'toggle-click-coordinates'], + ocr: ['change-match', 'toggle-click-coordinates', 'txtarea-refstr'], + exclude: ['toggle-click-coordinates'] + } + + if (!area || !(area.type in ctrlElemIds)) { + ctrlElemIds.all.forEach(elem => disableElem(elem)); + document.getElementById('txtarea-refstr').value = ''; + return; + } + ctrlElemIds[area.type].forEach(elem => enableElem(elem)); + const otherElemIds = ctrlElemIds.all.filter(elem => !ctrlElemIds[area.type].includes(elem)); + otherElemIds.forEach(elem => disableElem(elem)); + + if (area.type === 'ocr') { + document.getElementById('txtarea-refstr').value = area.refstr; + } else { + document.getElementById('txtarea-refstr').value = ''; + } +} + function addTag() { var input = $('#newtag'); var checkbox = nEditor.AddTag(input.val(), false); @@ -457,6 +522,10 @@ function setMatch() { nEditor.setMatch($('#match').val()); } +/** Method triggering update in refstr text box in needle editor instance. */ +function updateRefstr() { + nEditor.updateRefstr(); +} function toggleClickCoordinates() { updateToggleClickCoordinatesButton(nEditor.toggleClickCoordinates()); } diff --git a/lib/OpenQA/Task/Needle/Save.pm b/lib/OpenQA/Task/Needle/Save.pm index 4654d977c8e..d149eba9daa 100644 --- a/lib/OpenQA/Task/Needle/Save.pm +++ b/lib/OpenQA/Task/Needle/Save.pm @@ -31,8 +31,6 @@ sub _json_validation { if (!exists $djson->{tags} || !exists $djson->{tags}[0]) { die 'no tag defined'; } - my @not_ocr_area = grep { $_->{type} ne 'ocr' } @{$djson->{area}}; - die 'Cannot create a needle with only OCR areas' if scalar(@not_ocr_area) == 0; my $areas = $djson->{area}; foreach my $area (@$areas) { @@ -41,6 +39,11 @@ sub _json_validation { die 'area without type' unless exists $area->{type}; die 'area without height' unless exists $area->{height}; die 'area without width' unless exists $area->{width}; + if ($area->{type} eq 'ocr') { + die 'OCR area without refstr' unless defined $area->{refstr}; + die 'refstr is an empty string' if ($area->{refstr} eq ''); + die 'refstr contains placeholder char §' if ($area->{refstr} =~ qr/§/); + } } return $djson; } @@ -78,21 +81,25 @@ sub _save_needle { return $minion_job->finish({error => "Failed to validate $needlename.
$error"}); } - # determine imagepath + my @match_areas = grep { $_->{type} eq 'match' } @{$json_data->{area}}; + my $imagepath; - if ($imagedir) { - $imagepath = join('/', $imagedir, $imagename); - } - elsif ($imagedistri) { - $imagepath = join('/', needledir($imagedistri, $imageversion), $imagename); - } - else { - $imagepath = join('/', $openqa_job->result_dir(), $imagename); - } - if (!-f $imagepath) { - my $error = "Image $imagename could not be found!"; - $app->log->error("Failed to save needle: $error"); - return $minion_job->fail({error => "Failed to save $needlename.
$error"}); + if (@match_areas) { + # determine imagepath + if ($imagedir) { + $imagepath = join('/', $imagedir, $imagename); + } + elsif ($imagedistri) { + $imagepath = join('/', needledir($imagedistri, $imageversion), $imagename); + } + else { + $imagepath = join('/', $openqa_job->result_dir(), $imagename); + } + if (!-f $imagepath) { + my $error = "Image $imagename could not be found!"; + $app->log->error("Failed to save needle: $error"); + return $minion_job->fail({error => "Failed to save $needlename.
$error"}); + } } # check whether needle directory actually exists @@ -112,17 +119,19 @@ sub _save_needle { # do not overwrite the exist needle if disallow to overwrite my $baseneedle = "$needledir/$needlename"; - if (-e "$baseneedle.png" && !$args->{overwrite}) { + if (-e "$baseneedle.json" && !$args->{overwrite}) { #my $returned_data = $self->req->params->to_hash; #$returned_data->{requires_overwrite} = 1; return $minion_job->finish({requires_overwrite => 1}); } - # copy image my $success = 1; - if (!($imagepath eq "$baseneedle.png") && !copy($imagepath, "$baseneedle.png")) { - $app->log->error("Copy $imagepath -> $baseneedle.png failed: $!"); - $success = 0; + if (@match_areas) { + # copy image + if (!($imagepath eq "$baseneedle.png") && !copy($imagepath, "$baseneedle.png")) { + $app->log->error("Copy $imagepath -> $baseneedle.png failed: $!"); + $success = 0; + } } if ($success) { open(my $J, ">", "$baseneedle.json") or $success = 0; @@ -138,9 +147,11 @@ sub _save_needle { # commit needle in Git repository if ($git->enabled) { + my @files_to_be_comm = ["$needlename.json"]; + push(@files_to_be_comm, "$needlename.png") if (@match_areas); my $error = $git->commit( { - add => ["$needlename.json", "$needlename.png"], + add => @files_to_be_comm, message => ($commit_message || sprintf("%s for %s", $needlename, $openqa_job->name)), }); if ($error) { diff --git a/lib/OpenQA/WebAPI/Controller/Step.pm b/lib/OpenQA/WebAPI/Controller/Step.pm index e4ae52d9616..8d5623a9b24 100644 --- a/lib/OpenQA/WebAPI/Controller/Step.pm +++ b/lib/OpenQA/WebAPI/Controller/Step.pm @@ -287,7 +287,7 @@ sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_di my $pngfile = File::Spec->catpath('', $needles_dir, $png_fname); $needle->{needledir} = $needles_dir; - $needle->{image} = $pngfile; + $needle->{image} = $pngfile if (-f $pngfile); $needle->{json} = $file_name; $needle->{name} = $name; $needle->{distri} = $distri; @@ -315,12 +315,14 @@ sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $needle_info->{title} = $needle_name; $needle_info->{suggested_name} = ensure_timestamp_appended($needle_name); - $needle_info->{imageurl} - = $self->needle_url($distri, $needle_name . '.png', $version, $needle_info->{json})->to_string(); - $needle_info->{imagename} = basename($needle_info->{image}); - $needle_info->{imagedir} = dirname($needle_info->{image}); - $needle_info->{imagedistri} = $distri; - $needle_info->{imageversion} = $version; + if ($needle_info->{image}) { + $needle_info->{imageurl} + = $self->needle_url($distri, $needle_name . '.png', $version, $needle_info->{json})->to_string(); + $needle_info->{imagename} = basename($needle_info->{image}); + $needle_info->{imagedir} = dirname($needle_info->{image}); + $needle_info->{imagedistri} = $distri; + $needle_info->{imageversion} = $version; + } $needle_info->{tags} //= []; $needle_info->{matches} //= []; $needle_info->{properties} //= []; @@ -389,6 +391,10 @@ sub save_needle_ajax ($self) { my $job_id = $job->id; my $needledir = needledir($job->DISTRI, $job->VERSION); my $needlename = $validation->param('needlename'); + my $needle_json = $validation->param('json'); + + # The json data came from an HTML form. Might contain carriage return. + $needle_json =~ s/\r\n/\n/g; $self->gru->enqueue_and_keep_track( task_name => 'save_needle', @@ -396,7 +402,7 @@ sub save_needle_ajax ($self) { task_args => { job_id => $job_id, user_id => $self->current_user->id, - needle_json => $validation->param('json'), + needle_json => $needle_json, overwrite => $self->param('overwrite'), imagedir => $self->param('imagedir') // '', imagedistri => $validation->param('imagedistri'), diff --git a/templates/webapi/step/edit.html.ep b/templates/webapi/step/edit.html.ep index ad4ab1b9052..7f6dc83033d 100644 --- a/templates/webapi/step/edit.html.ep +++ b/templates/webapi/step/edit.html.ep @@ -220,6 +220,7 @@ Take image from: