Merge branch 'OpenDroneMap-master'

.dockerignore

@@ -1,3 +1,4 @@
 **/.git
 app/static/app/bundles
 **/node_modules
+.secret_key

.env

@@ -14,3 +14,4 @@ WO_DEFAULT_NODES=1
 WO_AUTH0_KEY=be8iHLsWn2t6ZsyZh0UofW1oaWScfsfC
 WO_AUTH0_DOMAIN=au-scalable-drone-cloud.au.auth0.com
 PIPELINES_URL="https://raw.githubusercontent.com/AuScalableDroneCloud/pipelines-jupyter/main/pipelines.yaml"
+WO_SETTINGS=

.gitignore

@@ -108,4 +108,5 @@ celerybeat-schedule.db
 
 # Debian builds
 dpkg/build
-dpkg/deb
+dpkg/deb
+.secret_key

README.md

@@ -38,9 +38,11 @@ A user-friendly, commercial grade software for drone image processing. Generate
 
 Windows and macOS users can purchase an automated [installer](https://www.opendronemap.org/webodm/download#installer), which makes the installation process easier.
 
-To install WebODM manually, these steps should get you up and running:
+There's also a cloud-hosted version of WebODM available from [webodm.net](https://webodm.net).
 
-* Install the following applications (if they are not installed already):
+To install WebODM manually on your machine:
+
+* Install the following applications:
   - [Git](https://git-scm.com/downloads)
   - [Docker](https://www.docker.com/)
   - [Docker-compose](https://docs.docker.com/compose/install/)
@@ -234,9 +236,9 @@ WebODM by itself is just a user interface (see [below](#odm-nodeodm-webodm-what)
 
 Small customizations such as changing the application colors, name, logo, or addying custom CSS/HTML/Javascript can be performed directly from the Customize -- Brand/Theme panels within WebODM. No need to fork or change the code.
 
-More advanced customizations can be achieved by writing [plugins](https://github.com/OpenDroneMap/WebODM/tree/master/plugins). This is the preferred way to add new functionality to WebODM since it requires less effort than maintaining a separate fork. The plugin system features server-side [signals](https://github.com/OpenDroneMap/WebODM/blob/master/app/plugins/signals.py) that can be used to be notified of various events, a ES6/React build system, a dynamic [client-side API](https://github.com/OpenDroneMap/WebODM/tree/master/app/static/app/js/classes/plugins) for adding elements to the UI, a built-in data store, an async task runner, a GRASS engine, hooks to add menu items and functions to rapidly inject CSS, Javascript and Django views.
+More advanced customizations can be achieved by writing [plugins](https://github.com/OpenDroneMap/WebODM/tree/master/coreplugins). This is the preferred way to add new functionality to WebODM since it requires less effort than maintaining a separate fork. The plugin system features server-side [signals](https://github.com/OpenDroneMap/WebODM/blob/master/app/plugins/signals.py) that can be used to be notified of various events, a ES6/React build system, a dynamic [client-side API](https://github.com/OpenDroneMap/WebODM/tree/master/app/static/app/js/classes/plugins) for adding elements to the UI, a built-in data store, an async task runner, a GRASS engine, hooks to add menu items and functions to rapidly inject CSS, Javascript and Django views.
 
-For plugins, the best source of documentation currently is to look at existing [code](https://github.com/OpenDroneMap/WebODM/tree/master/plugins). If a particular hook / entrypoint for your plugin does not yet exist, [request it](https://github.com/OpenDroneMap/WebODM/issues). We are adding hooks and entrypoints as we go.
+For plugins, the best source of documentation currently is to look at existing [code](https://github.com/OpenDroneMap/WebODM/tree/master/coreplugins). If a particular hook / entrypoint for your plugin does not yet exist, [request it](https://github.com/OpenDroneMap/WebODM/issues). We are adding hooks and entrypoints as we go.
 
 To create a plugin simply copy the `plugins/test` plugin into a new directory (for example, `plugins/myplugin`), then modify `manifest.json`, `plugin.py` and issue a `./webodm.sh restart`.
 
@@ -270,6 +272,7 @@ There are many ways to contribute back to the project:
  - Help answer questions on the community [forum](http://community.opendronemap.org/c/webodm) and [chat](https://gitter.im/OpenDroneMap/web-development).
  - ⭐️ us on GitHub.
  - Help us [translate](#translations) WebODM in your language.
+ - Help us classify [point cloud datasets](https://github.com/OpenDroneMap/ODMSemantic3D).
  - Spread the word about WebODM and OpenDroneMap on social media.
  - While we don't accept donations, you can purchase an [installer](https://webodm.org/download#installer), a [book](https://odmbook.com/) or a [sponsor package](https://github.com/users/pierotofy/sponsorship).
  - You can [pledge funds](https://fund.webodm.org) for getting new features built and bug fixed.

app/admin.py

@@ -10,10 +10,13 @@
 from django.urls import reverse
 from django.utils.html import format_html
 from guardian.admin import GuardedModelAdmin
+from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
+from django.contrib.auth.models import User
 
 from app.models import PluginDatum
 from app.models import Preset
 from app.models import Plugin
+from app.models import Profile
 from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin, valid_plugin, \
     get_plugins_persistent_path, clear_plugins_cache, init_plugins
 from .models import Project, Task, Setting, Theme
@@ -37,9 +40,9 @@ class TaskAdmin(admin.ModelAdmin):
     def has_add_permission(self, request):
         return False
 
-    list_display = ('id', 'project', 'processing_node', 'created_at', 'status', 'last_error')
+    list_display = ('id', 'name', 'project', 'processing_node', 'created_at', 'status', 'last_error')
     list_filter = ('status', 'project',)
-    search_fields = ('id', 'project__name')
+    search_fields = ('id', 'name', 'project__name')
 
 
 admin.site.register(Task, TaskAdmin)
@@ -260,3 +263,14 @@ def plugin_actions(self, obj):
 
 
 admin.site.register(Plugin, PluginAdmin)
+
+class ProfileInline(admin.StackedInline):
+    model = Profile
+    can_delete = False
+
+class UserAdmin(BaseUserAdmin):
+    inlines = [ProfileInline]
+
+# Re-register UserAdmin
+admin.site.unregister(User)
+admin.site.register(User, UserAdmin)

app/api/admin.py

@@ -1,7 +1,10 @@
 from django.contrib.auth.models import User, Group
-from rest_framework import serializers, viewsets, generics, status
+from app.models import Profile
+from rest_framework import serializers, viewsets, generics, status, exceptions
+from rest_framework.decorators import action
 from rest_framework.permissions import IsAdminUser
 from rest_framework.response import Response
+from django.core.exceptions import ObjectDoesNotExist
 from django.contrib.auth.hashers import make_password
 from app import models
 
@@ -20,6 +23,7 @@ def get_queryset(self):
         if email is not None:
             queryset = queryset.filter(email=email)
         return queryset
+
     def create(self, request):
         data = request.data.copy()
         password = data.get('password')
@@ -44,3 +48,37 @@ def get_queryset(self):
         if name is not None:
             queryset = queryset.filter(name=name)
         return queryset
+
+
+class ProfileSerializer(serializers.ModelSerializer):
+    class Meta:
+        model  = Profile
+        exclude = ('id', ) 
+
+        read_only_fields = ('user', )
+
+class AdminProfileViewSet(viewsets.ModelViewSet):
+    pagination_class = None
+    serializer_class = ProfileSerializer
+    permission_classes = [IsAdminUser]
+    lookup_field = 'user'
+
+    def get_queryset(self):
+        return Profile.objects.all()
+
+
+    @action(detail=True, methods=['post'])
+    def update_quota_deadline(self, request, user=None):
+        try:
+            hours = float(request.data.get('hours', ''))
+            if hours < 0:
+                raise ValueError("hours must be >= 0")
+        except ValueError as e:
+            raise exceptions.ValidationError(str(e))
+
+        try:
+            p = Profile.objects.get(user=user)
+        except ObjectDoesNotExist:
+            raise exceptions.NotFound()
+
+        return Response({'deadline': p.set_quota_deadline(hours)}, status=status.HTTP_200_OK)

app/api/externalauth.py

@@ -0,0 +1,39 @@
+from django.contrib.auth.models import User
+from django.contrib.auth import login
+from rest_framework.views import APIView
+from rest_framework import exceptions, permissions, parsers
+from rest_framework.response import Response
+from app.auth.backends import get_user_from_external_auth_response
+import requests
+from webodm import settings
+
+class ExternalTokenAuth(APIView):
+    permission_classes = (permissions.AllowAny,)
+    parser_classes = (parsers.JSONParser, parsers.FormParser,)
+
+    def post(self, request):
+        # This should never happen
+        if settings.EXTERNAL_AUTH_ENDPOINT == '':
+            return Response({'error': 'EXTERNAL_AUTH_ENDPOINT not set'})
+
+        token = request.COOKIES.get('external_access_token', '')
+        if token == '':
+            return Response({'error': 'external_access_token cookie not set'})
+
+        try:
+            r = requests.post(settings.EXTERNAL_AUTH_ENDPOINT, headers={
+                'Authorization': "Bearer %s" % token 
+            })
+            res = r.json()
+            if res.get('user_id') is not None:
+                user = get_user_from_external_auth_response(res)
+                if user is not None:
+                    login(request, user, backend='django.contrib.auth.backends.ModelBackend')
+                    return Response({'redirect': '/'})
+                else:
+                    return Response({'error': 'Invalid credentials'})
+            else:
+                return Response({'error': res.get('message', 'Invalid external server response')})
+        except Exception as e:
+            return Response({'error': str(e)})
+

app/api/formulas.py

@@ -54,10 +54,6 @@
         'expr': '(2 * G) - (R + B)',
         'help': _('Excess Green Index (derived from only the RGB bands) emphasizes the greenness of leafy crops such as potatoes.')
     },
-    'TGI': {
-        'expr': '(G - 0.39) * (R - 0.61) * B',
-        'help': _('Triangular Greenness Index (derived from only the RGB bands) performs similarly to EXG but with improvements over certain environments.')
-    },
     'BAI': {
         'expr': '1.0 / (((0.1 - R) ** 2) + ((0.06 - N) ** 2))',
         'help': _('Burn Area Index hightlights burned land in the red to near-infrared spectrum.')
@@ -144,16 +140,20 @@
     'NRB',
 
     'RGBN',
+    'RGNRe',
     'GRReN',
 
+    'RGBNRe',
     'BGRNRe',
     'BGRReN',
-    'RGBNRe',
     'RGBReN',
 
+    'RGBNReL',
     'BGRNReL',
     'BGRReNL',
 
+    'RGBNRePL',
+
     'L', # FLIR camera has a single LWIR band
 
     # more?
@@ -169,7 +169,7 @@ def lookup_formula(algo, band_order = 'RGB'):
 
     if algo not in algos:
         raise ValueError("Cannot find algorithm " + algo)
-
+    
     input_bands = tuple(b for b in re.split(r"([A-Z][a-z]*)", band_order) if b != "")
 
     def repl(matches):
@@ -191,7 +191,7 @@ def get_algorithm_list(max_bands=3):
         if k.startswith("_"):
             continue
 
-        cam_filters = get_camera_filters_for(algos[k], max_bands)
+        cam_filters = get_camera_filters_for(algos[k]['expr'], max_bands)
 
         if len(cam_filters) == 0:
             continue
@@ -204,9 +204,9 @@ def get_algorithm_list(max_bands=3):
 
     return res
 
-def get_camera_filters_for(algo, max_bands=3):
+@lru_cache(maxsize=100)
+def get_camera_filters_for(expr, max_bands=3):
     result = []
-    expr = algo['expr']
     pattern = re.compile("([A-Z]+?[a-z]*)")
     bands = list(set(re.findall(pattern, expr)))
     for f in camera_filters:
@@ -224,3 +224,45 @@ def get_camera_filters_for(algo, max_bands=3):
 
     return result
 
+@lru_cache(maxsize=1)
+def get_bands_lookup():
+    bands_aliases = {
+        'R': ['red', 'r'],
+        'G': ['green', 'g'],
+        'B': ['blue', 'b'],
+        'N': ['nir', 'n'],
+        'Re': ['rededge', 're'],
+        'P': ['panchro', 'p'],
+        'L': ['lwir', 'l']
+    }
+    bands_lookup = {}
+    for band in bands_aliases:
+        for a in bands_aliases[band]:
+            bands_lookup[a] = band
+    return bands_lookup
+
+def get_auto_bands(orthophoto_bands, formula):
+    algo = algos.get(formula)
+    if not algo:
+        raise ValueError("Cannot find formula: " + formula)
+
+    max_bands = len(orthophoto_bands) - 1 # minus alpha
+    filters = get_camera_filters_for(algo['expr'], max_bands)
+    if not filters:
+        raise valueError(f"Cannot find filters for {algo} with max bands {max_bands}")
+
+    bands_lookup = get_bands_lookup()
+    band_order = ""
+
+    for band in orthophoto_bands:
+        if band['name'] == 'alpha' or (not band['description']):
+            continue
+        f_band = bands_lookup.get(band['description'].lower())
+
+        if f_band is not None:
+            band_order += f_band
+
+    if band_order in filters:
+        return band_order, True
+    else:
+        return filters[0], False # Fallback

app/api/processingnodes.py

@@ -6,7 +6,7 @@
 from rest_framework.views import APIView
 
 from nodeodm.models import ProcessingNode
-
+from webodm import settings
 
 class ProcessingNodeSerializer(serializers.ModelSerializer):
     online = serializers.SerializerMethodField()
@@ -49,6 +49,18 @@ class ProcessingNodeViewSet(viewsets.ModelViewSet):
     serializer_class = ProcessingNodeSerializer
     queryset = ProcessingNode.objects.all()
 
+    def list(self, request, *args, **kwargs):
+        queryset = self.filter_queryset(self.get_queryset())
+
+        if settings.UI_MAX_PROCESSING_NODES is not None:
+            queryset = queryset[:settings.UI_MAX_PROCESSING_NODES]
+
+        if settings.NODE_OPTIMISTIC_MODE:
+            for pn in queryset:
+                pn.update_node_info()
+
+        serializer = self.get_serializer(queryset, many=True)
+        return Response(serializer.data)
 
 class ProcessingNodeOptionsView(APIView):
     """

app/api/projects.py

@@ -197,10 +197,12 @@ def edit(self, request, pk=None):
         return Response({'success': True}, status=status.HTTP_200_OK)
 
     def destroy(self, request, pk=None):
-        project = get_and_check_project(request, pk, ('delete_project', ))
+        project = get_and_check_project(request, pk, ('view_project', ))
 
         # Owner? Delete the project
         if project.owner == request.user or request.user.is_superuser:
+            get_and_check_project(request, pk, ('delete_project', ))
+
             return super().destroy(self, request, pk=pk)
         else:
             # Do not remove the project, simply remove all user's permissions to the project