Skip to content

Commit

Permalink
[AJP-3] Introduce AI predictions
Browse files Browse the repository at this point in the history
  • Loading branch information
Anna Grund authored and Anna Grund committed Oct 14, 2023
1 parent ef7a0a7 commit f5b285f
Show file tree
Hide file tree
Showing 10 changed files with 349 additions and 6 deletions.
57 changes: 57 additions & 0 deletions ajapaik/ajapaik/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2074,6 +2074,63 @@ class PhotoViewpointElevationSuggestion(Suggestion):
proposer = ForeignKey('Profile', blank=True, null=True, related_name='photo_viewpoint_elevation_suggestions',
on_delete=CASCADE)

class PhotoModelSuggestionResult(Suggestion):
INTERIOR, EXTERIOR = range(2)
GROUND_LEVEL, RAISED, AERIAL = range(3)
SCENE_CHOICES = (
(INTERIOR, _('Interior')),
(EXTERIOR, _('Exterior'))
)
VIEWPOINT_ELEVATION_CHOICES = (
(GROUND_LEVEL, _('Ground')),
(RAISED, _('Raised')),
(AERIAL, _('Aerial'))
)
viewpoint_elevation = PositiveSmallIntegerField(_('Viewpoint elevation'), choices=VIEWPOINT_ELEVATION_CHOICES, blank=True, null=True)
scene = PositiveSmallIntegerField(_('Scene'), choices=SCENE_CHOICES, blank=True, null=True)


class PhotoModelSuggestionAlternativeCategory(Suggestion):
INTERIOR, EXTERIOR = range(2)
GROUND_LEVEL, RAISED, AERIAL = range(3)
SCENE_CHOICES = (
(INTERIOR, _('Interior')),
(EXTERIOR, _('Exterior'))
)
VIEWPOINT_ELEVATION_CHOICES = (
(GROUND_LEVEL, _('Ground')),
(RAISED, _('Raised')),
(AERIAL, _('Aerial'))
)
viewpoint_elevation_alternation = PositiveSmallIntegerField(_('Viewpoint elevation'),
choices=VIEWPOINT_ELEVATION_CHOICES, blank=True,
null=True)
scene_alternation = PositiveSmallIntegerField(_('Scene'), choices=SCENE_CHOICES, blank=True, null=True)

proposer = ForeignKey('Profile', blank=True, null=True, related_name='photo_scene_suggestions_alternation',
on_delete=CASCADE)

def validate_unique(self, exclude=None):
# super().validate_unique(exclude)
queryset = self.__class__._default_manager.filter(
Q(scene_alternation=0) | Q(scene_alternation=1),
proposer=self.proposer,
photo_id=self.photo_id
).exclude(pk=self.pk)

if self.scene_alternation in ['0', '1'] and queryset.exists():
return False
return True

def save(self, *args, **kwargs):
if self.validate_unique():
super().save(*args, **kwargs)

class Meta:
db_table = 'ajapaik_photomodelsuggestionalternativecategory'
unique_together = (('proposer', 'photo_id', 'scene_alternation'),
('proposer', 'photo_id', 'viewpoint_elevation_alternation'))


class PhotoFlipSuggestion(Suggestion):
proposer = ForeignKey('Profile', blank=True, null=True, related_name='photo_flip_suggestions', on_delete=CASCADE)
Expand Down
72 changes: 72 additions & 0 deletions ajapaik/ajapaik/static/js/ajp-category-suggestion.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
function submitCategorySuggestion(photoIds, isMultiSelect) {
sendCategorySuggestionToAI(photoIds, scene, viewpointElevation)
$('#ajp-loading-overlay').show();
return fetch(photoSceneUrl, {
method: 'POST',
Expand Down Expand Up @@ -99,3 +100,74 @@ function clickSceneCategoryButton(buttonId) {
$('#' + buttonId).removeClass('btn-outline-dark');
$('#' + buttonId).removeClass('btn-light');
}

function getImageCategory(photoId, callback) {
let onSuccess = function (response) {
callback(determinePictureCategory(response.data));
};
getRequest(
'/object-categorization/get-latest-category/' + photoId + '/',
null,
null,
constants.translations.queries.GET_CATEGORY_FAILED,
onSuccess
);
}

function determinePictureCategory(responseData) {
let responseDict = {};
for (let data of responseData) {
let fields = data["fields"];
if ("scene" in fields) {
if (fields["scene"] === 0) {
responseDict["scene"] = "interior";
} else {
responseDict["scene"] = "exterior";
}
}
if ("viewpoint_elevation" in fields) {
if (fields["viewpoint_elevation"] === 0) {
responseDict["viewpoint_elevation"] = "ground";
} else if (fields["viewpoint_elevation"] === 1) {
responseDict["viewpoint_elevation"] = "raised";
} else if (fields["viewpoint_elevation"] === 2) {
responseDict["viewpoint_elevation"] = "areal";
}
}
}
return responseDict;
}

function sendCategorySuggestionToAI(photoIds, scene, viewpointElevation) {
let sceneVerdict = scene.toLowerCase();
let viewpointElevationVerdict = viewpointElevation.toLowerCase();

let payload = {
"photo_id": photoIds[0]
};

if (sceneVerdict === "interior") {
payload["scene_to_alternate"] = 0
}
if (sceneVerdict === "exterior") {
payload["scene_to_alternate"] = 1
}
if (viewpointElevationVerdict === "ground") {
payload["viewpoint_elevation_to_alternate"] = 0
}
if (viewpointElevationVerdict === "raised") {
payload["viewpoint_elevation_to_alternate"] = 1
}
if (viewpointElevationVerdict === "raised") {
payload["viewpoint_elevation_to_alternate"] = 2
}

postRequest(
'/object-categorization/confirm-latest-category',
payload,
constants.translations.queries.POST_CATEGORY_CONFIRMATION_SUCCESS,
constants.translations.queries.POST_CATEGORY_CONFIRMATION_FAILED,
function () {
}
);
}
7 changes: 7 additions & 0 deletions ajapaik/ajapaik/static/js/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ var constants = {
},
},
queries: {
GET_CATEGORY_FAILED: gettext('Failed to load object category'),
POST_CATEGORY_CONFIRMATION_SUCCESS: gettext(
'Successfully posted object category confirmation'
),
POST_CATEGORY_CONFIRMATION_FAILED: gettext(
'Failed to post object category confirmation'
),
GET_ANNOTATION_CLASSES_FAILED: gettext(
'Failed to load object annotation classes'
),
Expand Down
73 changes: 71 additions & 2 deletions ajapaik/ajapaik/templates/_toolbox.html
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,9 @@
</ul>
{% trans 'Share' %}
</div>
<div class="d-flex mr-2 ajp-toolbox-button-wrapper"
<div class="d-flex mr-2 ajp-toolbox-button-wrapper" style="display:grid; position: relative;"
title="{% trans 'Add a category - clicking this button will enable you to categorize this picture to various categories such as whether this picture depicts indoors or outdoors' %}">
<div style='position: absolute; top: 0; right: 0; background-color: #cfcdcc; color: #f2683a; font-size: 12px; padding: 1px 2px;'>AI</div>
<button id="ajp-categorize-scene" type="button" data-toggle="popover" class="ajp-button" tabindex="0"
data-is-categorization-button="true">
<span class="material-icons notranslate ajp-text-gray ajp-icon-36">category</span>
Expand Down Expand Up @@ -147,6 +148,35 @@
var shareUrl = '{{ hostname }}{% if is_photo_modal %}{{ photo.get_detail_url }}{% elif rephoto %}{{ rephoto.get_absolute_url }}{% else %}{{ photo.get_absolute_url }}{% endif %}';
window.lastEnteredName = undefined;

function handleAIPrediction() {
getImageCategory('{{ photo.id }}', function (categoryMap) {
console.log(categoryMap)
let scene = categoryMap["scene"]
let viewpoint_elevation = categoryMap["viewpoint_elevation"]

if (scene === "exterior") {
clickSceneCategoryButton('exterior-button');
$("#exterior-ai").show();
}
if (scene === "interior") {
clickSceneCategoryButton('interior-button');
$("#interior-ai").show();
}
if (viewpoint_elevation === "ground") {
clickViewpointElevationCategoryButton('ground-button');
$("#ground-ai").show();
}
if (viewpoint_elevation === "aerial") {
clickViewpointElevationCategoryButton('aerial-button');
$("#aerial-ai").show();
}
if (viewpoint_elevation === "raised") {
clickViewpointElevationCategoryButton('raised-button');
$("#raised-ai").show();
}
});
}

function fetchPhotoInfo(url) {
return fetch(url + '{{ photo.id }}' + '/').then(function(response) {
return response.json();
Expand Down Expand Up @@ -209,7 +239,45 @@
buttonTranslation = 'Submit';
}

let content = `<div class='d-flex mb-4 justify-content-center'><div class='d-flex' style='flex-direction:column;align-items:center;'><button onclick='clickSceneCategoryButton(this.id);' id='interior-button' class='btn mr-2 ` + topLeftButtonClass + `' style='display:grid;'><span class='material-icons notranslate ajp-icon-48'>hotel</span><span>` + gettext('Interior') + `</span></button></div><div class='d-flex' style='flex-direction:column;align-items:center;'><button onclick='clickSceneCategoryButton(this.id);' id='exterior-button' class='btn ml-2 ` + topRightButtonClass + `' style='display:grid;'><span class='material-icons ajp-icon-48 notranslate'>home</span><span>` + gettext('Exterior') + `</span></button></div></div><div class='d-flex'><div class='d-flex' style='flex-direction:column;align-items:center;'><button onclick='clickViewpointElevationCategoryButton(this.id);' id='ground-button' class='btn mr-2 ` + bottomLeftButtonClass + `' style='display:grid;'><span class='material-icons notranslate ajp-icon-48'>nature_people</span><span>` + gettext('Ground') + `</span></button></div><div class='d-flex' style='flex-direction:column;align-items:center;'><button onclick='clickViewpointElevationCategoryButton(this.id);' id='raised-button' class='btn mr-2 ` + bottomMiddleButtonClass + `' style='display:grid;'><span class='material-icons notranslate ajp-icon-48'>location_city</span><span>` + gettext('Raised') + `</span></button></div><div class='d-flex' style='flex-direction:column;align-items:center;'><button onclick='clickViewpointElevationCategoryButton(this.id);' id='aerial-button' class='btn ml-2 d-grid ` + bottomRightButtonClass + `' style='display:grid;'><span class='material-icons ajp-icon-48 notranslate'>flight</span><span>` + gettext('Aerial') + `</span></button></div></div>`;
let content = `<div class='d-flex mb-4 justify-content-center'>
<div class='d-flex' style='flex-direction:column;align-items:center;'>
<button onclick='clickSceneCategoryButton(this.id);' id='interior-button' class='btn mr-2 ` + topLeftButtonClass + `' style='display:grid; position: relative;'>
<div id='interior-ai' style='display: none; position: absolute; top: 0; right: 0; background-color: #cfcdcc; color: #f2683a; font-size: 12px; padding: 1px 2px;'>AI</div>
<span class='material-icons notranslate ajp-icon-48'>hotel</span>
<span>` + gettext('Interior') + `</span>
</button>
</div>
<div class='d-flex' style='flex-direction:column;align-items:center;'>
<button onclick='clickSceneCategoryButton(this.id);' id='exterior-button' class='btn ml-2 ` + topRightButtonClass + `' style='display:grid; position: relative;'>
<div id='exterior-ai' style='display: none; position: absolute; top: 0; right: 0; background-color: #cfcdcc; color: #f2683a; font-size: 12px; padding: 1px 2px;'>AI</div>
<span class='material-icons ajp-icon-48 notranslate'>home</span>
<span>` + gettext('Exterior') + `</span>
</button>
</div>
</div>
<div class='d-flex'><div class='d-flex' style='flex-direction:column;align-items:center;'>
<button onclick='clickViewpointElevationCategoryButton(this.id);' id='ground-button' class='btn mr-2 ` + bottomLeftButtonClass + `' style='display:grid; position: relative;'>
<div id='ground-ai' style='display: none; position: absolute; top: 0; right: 0; background-color: #cfcdcc; color: #f2683a; font-size: 12px; padding: 1px 2px;'>AI</div>
<span class='material-icons notranslate ajp-icon-48'>nature_people</span>
<span>` + gettext('Ground') + `</span>
</button>
</div>
<div class='d-flex' style='flex-direction:column;align-items:center;'>
<button onclick='clickViewpointElevationCategoryButton(this.id);' id='raised-button' class='btn mr-2 ` + bottomMiddleButtonClass + `' style='display:grid; position: relative;'>
<div id='raised-ai' style='display: none; position: absolute; top: 0; right: 0; background-color: #cfcdcc; color: #f2683a; font-size: 12px; padding: 1px 2px;'>AI</div>
<span class='material-icons notranslate ajp-icon-48'>location_city</span>
<span>` + gettext('Raised') + `</span>
</button>
</div>
<div class='d-flex' style='flex-direction:column;align-items:center;'>
<button onclick='clickViewpointElevationCategoryButton(this.id);' id='aerial-button' class='btn ml-2 d-grid ` + bottomRightButtonClass + `' style='display:grid; position: relative;'>
<div id='aerial-ai' style='display: none; position: absolute; top: 0; right: 0; background-color: #cfcdcc; color: #f2683a; font-size: 12px; padding: 1px 2px;'>AI</div>
<span class='material-icons ajp-icon-48 notranslate'>flight</span>
<span>` + gettext('Aerial') + `</span>
</button>
</div>
</div>`;

let actionButtonTemplate = `<button id='send-suggestion-button' onclick="submitCategorySuggestion(['{{ photo.id }}'], false);" class='btn btn-success mt-3 w-100' disabled>` + gettext(buttonTranslation) + `</button>`;
content += actionButtonTemplate;

Expand Down Expand Up @@ -314,6 +382,7 @@
$('#ajp-categorize-scene').popover('hide');
} else {
$('#ajp-categorize-scene').popover('show');
handleAIPrediction();
}
});

Expand Down
4 changes: 3 additions & 1 deletion ajapaik/ajapaik/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from ajapaik.ajapaik.sitemaps import PhotoSitemap, StaticViewSitemap
from ajapaik.ajapaik_face_recognition import urls as fr_urls
from ajapaik.ajapaik_object_recognition import urls as or_urls
from ajapaik.ajapaik_object_categorization import urls as oc_urls

urlpatterns = [
url(r'^stream/', views.fetch_stream, name='fetch_stream'),
Expand Down Expand Up @@ -222,7 +223,8 @@
url(r'^sitemap-(?P<section>.+).xml$', cache_page(86400)(sitemap_views.sitemap), {'sitemaps': sitemaps},
name='django.contrib.sitemaps.views.sitemap'),
url(r'^face-recognition/', include(fr_urls)),
url(r'^object-recognition/', include(or_urls))
url(r'^object-recognition/', include(or_urls)),
url(r'^object-categorization/', include(oc_urls)) #TODO verify
]

if hasattr(settings, 'GOOGLE_ANALYTICS_KEY') and settings.GOOGLE_ANALYTICS_KEY == 'UA-21689048-1':
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import json

from django.http import HttpRequest

from ajapaik.ajapaik.models import Photo, PhotoModelSuggestionResult, \
PhotoModelSuggestionAlternativeCategory
from django.core import serializers
import datetime


def get_latest_category_from_result_table(photo_id=None):
result = PhotoModelSuggestionResult.objects.filter(photo_id=photo_id).order_by('-created')
result_categories = []

if result.exists():
categories = serializers.serialize('python', result)
result_categories.append(categories)
return categories
return result_categories


def get_uncategorized_photos():
photos = Photo.objects.filter(scene=None, viewpoint_elevation=None).values_list('id', 'user', 'image')

photos_without_results = []
for photo in photos:
if not PhotoModelSuggestionResult.objects.filter(photo_id=photo[0]).exists():
photos_without_results.append(photo)

return photos_without_results


def post_image_category_result_table(request: HttpRequest):
result = PhotoModelSuggestionResult()
data = json.loads(request.body)

result.viewpoint_elevation = data.get("verdict_view_point_elevation", None)
result.scene = data.get("verdict_scene", None)
result.photo_id = data.get("photo_id", None)
result.save()
return result


def aggregate_category_data():
one_week_ago = datetime.datetime.now() - datetime.timedelta(days=7)

alternative_category = PhotoModelSuggestionAlternativeCategory.objects.filter(created__gte=one_week_ago).order_by(
'-created')

result_categories = {}

if alternative_category.exists():
confirm_reject_dict = serializers.serialize('python', alternative_category)
result_categories['alternative_category_data'] = confirm_reject_dict

return result_categories


class UncategorizedPhoto:
def __init__(self, image_id, user_id, image_name):
self.image_id = image_id
self.user_id = user_id
self.image_name = image_name
16 changes: 16 additions & 0 deletions ajapaik/ajapaik_object_categorization/urls.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from django.conf.urls import url

from ajapaik.ajapaik_object_categorization import views

urlpatterns = [
url(r'^get-latest-category/(?P<photo_id>\d+)/$', views.get_latest_category_from_result_table,
name='picture_recognition_get_latest_category'),
url(r'^confirm-latest-category', views.propose_alternative_category,
name='picture_recognition_get_latest_category'),
url(r'^publish-picture-category-result', views.post_image_category_result_table,
name='publish_picture_category_result'),
url(r'aggregate-category-data', views.aggregate_category_data,
name='aggregate_category_data'),
url(r'get-uncategorized-images', views.get_uncategorized_photos,
name='get_uncategorized_images')
]
Loading

0 comments on commit f5b285f

Please sign in to comment.