Skip to content

Commit

Permalink
Improve UX of OCR-recognized images
Browse files Browse the repository at this point in the history
Rather than dumping the JSON, let's output a USWDS style infobox with
the contents rendered a bit nicer for non-techies.

This also institutes an arbitrary "45" threshold of confidence to
consider the OCR result "legible". Although people on the internet say
that confidence really should be 95+, this is a lower bound that will
hopefully not raise too many false positives.
  • Loading branch information
tdooner committed Mar 19, 2024
1 parent daf6db4 commit 5d1ec7b
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 10 deletions.
4 changes: 3 additions & 1 deletion app/src/pages/api/upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,10 @@ const ocrDetectionAction = async (
) as unknown as PromiseRejectedResult[];

const fulfilled = fulfilledFiles.map((result) => {
// don't send the image data to the client
// don't send the image data to the client, but keep the filename
result.value.data.map((doc) => {
const fileName = doc.image.fileName;
doc.fileName = fileName;
delete doc.image;
return doc;
});
Expand Down
55 changes: 46 additions & 9 deletions app/src/pages/upload/confirmation.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { ParsingFunctionResult } from "@/service/ocr/parser";
import {
Alert,
IconList,
IconListContent,
IconListItem,
Expand All @@ -17,6 +19,8 @@ type ParsedBlurDecetorResults = ResponseData & {
results: BlurryDetectorResults;
};

const BLURRINESS_THRESHOLD = 45;

const formatImagePath = (path: string) => {
return (path.match("([^/]+$)") || [])[0];
};
Expand All @@ -25,6 +29,11 @@ const createBlurrinessIncidatorText = (isBlurry: boolean) => {
return isBlurry ? " is blurry." : " successfully uploaded.";
};

const createLegibileIncidatorText = (isBlurry: boolean) => {
return isBlurry ? " may be too blurry to read." : " successfully uploaded.";
};


const createBlurrinessIncidatorIcon = (isBlurry: boolean) => {
return (
<>
Expand All @@ -48,6 +57,30 @@ const createBlurrinessIncidatorIcon = (isBlurry: boolean) => {
);
};

const renderDocumentFields = (docFields: ParsingFunctionResult) => {
// ignore the top-level keys, just show an alert if there is any
// data in the second level
const recognizedData = Object.entries(docFields).map(([_docType, fields]) => {
return Object.entries(fields).filter(([_, v]) => v != "");
}).flat();

if (recognizedData.length > 0) {
return (
<Alert
type="success"
heading="We recognized text in your document!"
headingLevel="h4"
>
<strong>For demo purposes:</strong>{' '}
<span>We were able to recognize the following fields, which we could send to the caseworker:</span>
<ul>
{recognizedData.map(([field, value], i) => <li key={i}><strong>{field}</strong>: {value}</li>)}
</ul>
</Alert>
);
}
};

const renderOcrResults = (results: OCRDectionResponse) => {
const elements = results.fulfilled.map((documentResults, index) => {
// get the orientation with the highest confidence
Expand All @@ -58,18 +91,22 @@ const renderOcrResults = (results: OCRDectionResponse) => {
(doc) => doc.confidence === highestConfidenceScore
)[0];
const docFields = highestConfidenceOrientation.documents;
const isIllegible = highestConfidenceOrientation.confidence < BLURRINESS_THRESHOLD;

return (
<div key={`document-${index}`}>
<h3>Document {index + 1}</h3>
<p>
<strong>Confidence:</strong> {highestConfidenceOrientation.confidence}
</p>
<p>
<strong>Rotation:</strong>{" "}
{highestConfidenceOrientation.rotatedOrientation}
</p>
{JSON.stringify(docFields, null, 2)}
<IconListItem key={index} className="usa-icon-list__item">
<>
{createBlurrinessIncidatorIcon(isIllegible)}
<IconListContent>
{" "}
{formatImagePath(highestConfidenceOrientation.fileName)}
{createLegibileIncidatorText(isIllegible)}
{` (confidence: ${highestConfidenceScore})`}
</IconListContent>
</>
</IconListItem>
{renderDocumentFields(docFields)}
</div>
);
});
Expand Down
1 change: 1 addition & 0 deletions app/src/service/ocr/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export type OcrOptions = {
export type ProcessedImageResult = {
documents: ReturnType<typeof parse>;
image?: DocumentImage;
fileName: string;
percentages: Record<ParserKeys, number>;
confidence: number;
rotatedOrientation?: DocumentOrientation;
Expand Down

0 comments on commit 5d1ec7b

Please sign in to comment.