From cc816a66e9e7ff76ac5f84ddbb7f0c8a9fe6d948 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 30 May 2023 13:28:55 -0400 Subject: [PATCH 01/95] Mention ODMSemantic3D --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0dc1df5fc..976d2f145 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,7 @@ There are many ways to contribute back to the project: - Help answer questions on the community [forum](http://community.opendronemap.org/c/webodm) and [chat](https://gitter.im/OpenDroneMap/web-development). - ⭐️ us on GitHub. - Help us [translate](#translations) WebODM in your language. + - Help us classify [point cloud datasets](https://github.com/OpenDroneMap/ODMSemantic3D). - Spread the word about WebODM and OpenDroneMap on social media. - While we don't accept donations, you can purchase an [installer](https://webodm.org/download#installer), a [book](https://odmbook.com/) or a [sponsor package](https://github.com/users/pierotofy/sponsorship). - You can [pledge funds](https://fund.webodm.org) for getting new features built and bug fixed. From 3316d1c3a83ce8ddf4cbb9da83b9f1c488742616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Acu=C3=B1a?= <45692265+diegoaces@users.noreply.github.com> Date: Sat, 3 Jun 2023 17:26:22 -0400 Subject: [PATCH 02/95] New chart plugin --- coreplugins/projects-charts/__init__.py | 1 + coreplugins/projects-charts/disabled | 0 coreplugins/projects-charts/manifest.json | 16 +++ coreplugins/projects-charts/plugin.py | 93 +++++++++++++ .../projects-charts/public/Chart.min.js | 20 +++ .../projects-charts/templates/index.html | 126 ++++++++++++++++++ 6 files changed, 256 insertions(+) create mode 100644 coreplugins/projects-charts/__init__.py create mode 100644 coreplugins/projects-charts/disabled create mode 100644 coreplugins/projects-charts/manifest.json create mode 100644 coreplugins/projects-charts/plugin.py create mode 100644 coreplugins/projects-charts/public/Chart.min.js create mode 100644 coreplugins/projects-charts/templates/index.html diff --git a/coreplugins/projects-charts/__init__.py b/coreplugins/projects-charts/__init__.py new file mode 100644 index 000000000..48aad58ec --- /dev/null +++ b/coreplugins/projects-charts/__init__.py @@ -0,0 +1 @@ +from .plugin import * diff --git a/coreplugins/projects-charts/disabled b/coreplugins/projects-charts/disabled new file mode 100644 index 000000000..e69de29bb diff --git a/coreplugins/projects-charts/manifest.json b/coreplugins/projects-charts/manifest.json new file mode 100644 index 000000000..dba494ece --- /dev/null +++ b/coreplugins/projects-charts/manifest.json @@ -0,0 +1,16 @@ +{ + "name": "Projects and tasks charts Plugin", + "webodmMinVersion": "0.1.0", + "description": "A plugin to show charts of projects and tasks", + "version": "0.1.0", + "author": "Diego Acuña, Greenbot Labs", + "email": "dacuna@greenbot.cl", + "repository": "https://github.com/OpenDroneMap/WebODM", + "tags": [ + "plugin", + "charts" + ], + "homepage": "https://github.com/OpenDroneMap/WebODM", + "experimental": false, + "deprecated": false +} \ No newline at end of file diff --git a/coreplugins/projects-charts/plugin.py b/coreplugins/projects-charts/plugin.py new file mode 100644 index 000000000..3197f6339 --- /dev/null +++ b/coreplugins/projects-charts/plugin.py @@ -0,0 +1,93 @@ +from datetime import datetime + +from django import forms +from django.contrib.auth.decorators import login_required +from django.db.models import Count +from django.db.models.functions import TruncMonth +from django.shortcuts import render +from django.utils.translation import gettext as _ + +from app.models import Project, Task +from app.plugins import PluginBase, Menu, MountPoint + + +def get_first_year(): + project = Project.objects.order_by('created_at').first() + if project: + return project.created_at.year + else: + return datetime.now().year + + +def get_last_year(): + project = Project.objects.order_by('created_at').last() + if project: + return project.created_at.year + else: + return datetime.now().year + + +year_choices = [(r, r) for r in + range(get_first_year(), get_last_year() + 1)] + + +class ProjectForm(forms.Form): + year = forms.IntegerField(label='Year', + widget=forms.Select(choices=year_choices, attrs={'class': 'form-control'}, + )) + + +def get_projects_by_month(year=datetime.now().year): + return Project.objects.filter(created_at__year=year).annotate( + month=TruncMonth('created_at')).values('month').annotate( + c=Count('id')).values_list('month', 'c').order_by('month') + + +def get_tasks_by_month(year): + return Task.objects.filter(created_at__year=year).annotate( + month=TruncMonth('created_at')).values('month').annotate( + c=Count('id')).values_list('month', 'c').order_by('month') + + +class Plugin(PluginBase): + + def main_menu(self): + return [Menu(_("Charts"), self.public_url(""), "fa fa-chart-bar")] + + def include_js_files(self): + return ['Chart.min.js'] + + def app_mount_points(self): + @login_required + def index(request): + list_count_projects_by_month = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + list_count_tasks_by_month = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + year = datetime.now().year + form = ProjectForm(request.GET) + + if request.method == "GET": + if form.is_valid(): + date = request.GET.get('year') + year = date.split('-')[0] + else: + form = ProjectForm(initial={'year': year}) + + for i in get_projects_by_month(year): + list_count_projects_by_month.insert(i[0].month - 1, i[1]) + + for i in get_tasks_by_month(year): + list_count_tasks_by_month.insert(i[0].month - 1, i[1]) + + template_args = { + 'form': form, + 'projects_by_month': list_count_projects_by_month, + 'tasks_by_month': list_count_tasks_by_month, + 'title': 'Charts', + 'months': ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', + 'October', 'November', 'December'] + } + return render(request, self.template_path("index.html"), template_args) + + return [ + MountPoint('$', index) + ] diff --git a/coreplugins/projects-charts/public/Chart.min.js b/coreplugins/projects-charts/public/Chart.min.js new file mode 100644 index 000000000..d1489e559 --- /dev/null +++ b/coreplugins/projects-charts/public/Chart.min.js @@ -0,0 +1,20 @@ +/** + * Skipped minification because the original files appears to be already minified. + * Original file: /npm/chart.js@4.3.0/dist/chart.umd.js + * + * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files + */ +/*! + * Chart.js v4.3.0 + * https://www.chartjs.org + * (c) 2023 Chart.js Contributors + * Released under the MIT License + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):(t="undefined"!=typeof globalThis?globalThis:t||self).Chart=e()}(this,(function(){"use strict";var t=Object.freeze({__proto__:null,get Colors(){return Ko},get Decimation(){return Jo},get Filler(){return pa},get Legend(){return _a},get SubTitle(){return wa},get Title(){return va},get Tooltip(){return Va}});function e(){}const i=(()=>{let t=0;return()=>t++})();function s(t){return null==t}function n(t){if(Array.isArray&&Array.isArray(t))return!0;const e=Object.prototype.toString.call(t);return"[object"===e.slice(0,7)&&"Array]"===e.slice(-6)}function o(t){return null!==t&&"[object Object]"===Object.prototype.toString.call(t)}function a(t){return("number"==typeof t||t instanceof Number)&&isFinite(+t)}function r(t,e){return a(t)?t:e}function l(t,e){return void 0===t?e:t}const h=(t,e)=>"string"==typeof t&&t.endsWith("%")?parseFloat(t)/100:+t/e,c=(t,e)=>"string"==typeof t&&t.endsWith("%")?parseFloat(t)/100*e:+t;function d(t,e,i){if(t&&"function"==typeof t.call)return t.apply(i,e)}function u(t,e,i,s){let a,r,l;if(n(t))if(r=t.length,s)for(a=r-1;a>=0;a--)e.call(i,t[a],a);else for(a=0;at,x:t=>t.x,y:t=>t.y};function v(t){const e=t.split("."),i=[];let s="";for(const t of e)s+=t,s.endsWith("\\")?s=s.slice(0,-1)+".":(i.push(s),s="");return i}function M(t,e){const i=y[e]||(y[e]=function(t){const e=v(t);return t=>{for(const i of e){if(""===i)break;t=t&&t[i]}return t}}(e));return i(t)}function w(t){return t.charAt(0).toUpperCase()+t.slice(1)}const k=t=>void 0!==t,S=t=>"function"==typeof t,P=(t,e)=>{if(t.size!==e.size)return!1;for(const i of t)if(!e.has(i))return!1;return!0};function D(t){return"mouseup"===t.type||"click"===t.type||"contextmenu"===t.type}const C=Math.PI,O=2*C,A=O+C,T=Number.POSITIVE_INFINITY,L=C/180,E=C/2,R=C/4,I=2*C/3,z=Math.log10,F=Math.sign;function V(t,e,i){return Math.abs(t-e)t-e)).pop(),e}function W(t){return!isNaN(parseFloat(t))&&isFinite(t)}function H(t,e){const i=Math.round(t);return i-e<=t&&i+e>=t}function j(t,e,i){let s,n,o;for(s=0,n=t.length;sl&&h=Math.min(e,i)-s&&t<=Math.max(e,i)+s}function et(t,e,i){i=i||(i=>t[i]1;)s=o+n>>1,i(s)?o=s:n=s;return{lo:o,hi:n}}const it=(t,e,i,s)=>et(t,i,s?s=>{const n=t[s][e];return nt[s][e]et(t,i,(s=>t[s][e]>=i));function nt(t,e,i){let s=0,n=t.length;for(;ss&&t[n-1]>i;)n--;return s>0||n{const i="_onData"+w(e),s=t[e];Object.defineProperty(t,e,{configurable:!0,enumerable:!1,value(...e){const n=s.apply(this,e);return t._chartjs.listeners.forEach((t=>{"function"==typeof t[i]&&t[i](...e)})),n}})})))}function rt(t,e){const i=t._chartjs;if(!i)return;const s=i.listeners,n=s.indexOf(e);-1!==n&&s.splice(n,1),s.length>0||(ot.forEach((e=>{delete t[e]})),delete t._chartjs)}function lt(t){const e=new Set(t);return e.size===t.length?t:Array.from(e)}const ht="undefined"==typeof window?function(t){return t()}:window.requestAnimationFrame;function ct(t,e){let i=[],s=!1;return function(...n){i=n,s||(s=!0,ht.call(window,(()=>{s=!1,t.apply(e,i)})))}}function dt(t,e){let i;return function(...s){return e?(clearTimeout(i),i=setTimeout(t,e,s)):t.apply(this,s),e}}const ut=t=>"start"===t?"left":"end"===t?"right":"center",ft=(t,e,i)=>"start"===t?e:"end"===t?i:(e+i)/2,gt=(t,e,i,s)=>t===(s?"left":"right")?i:"center"===t?(e+i)/2:e;function pt(t,e,i){const s=e.length;let n=0,o=s;if(t._sorted){const{iScale:a,_parsed:r}=t,l=a.axis,{min:h,max:c,minDefined:d,maxDefined:u}=a.getUserBounds();d&&(n=J(Math.min(it(r,a.axis,h).lo,i?s:it(e,l,a.getPixelForValue(h)).lo),0,s-1)),o=u?J(Math.max(it(r,a.axis,c,!0).hi+1,i?0:it(e,l,a.getPixelForValue(c),!0).hi+1),n,s)-n:s-n}return{start:n,count:o}}function mt(t){const{xScale:e,yScale:i,_scaleRanges:s}=t,n={xmin:e.min,xmax:e.max,ymin:i.min,ymax:i.max};if(!s)return t._scaleRanges=n,!0;const o=s.xmin!==e.min||s.xmax!==e.max||s.ymin!==i.min||s.ymax!==i.max;return Object.assign(s,n),o}class bt{constructor(){this._request=null,this._charts=new Map,this._running=!1,this._lastDate=void 0}_notify(t,e,i,s){const n=e.listeners[s],o=e.duration;n.forEach((s=>s({chart:t,initial:e.initial,numSteps:o,currentStep:Math.min(i-e.start,o)})))}_refresh(){this._request||(this._running=!0,this._request=ht.call(window,(()=>{this._update(),this._request=null,this._running&&this._refresh()})))}_update(t=Date.now()){let e=0;this._charts.forEach(((i,s)=>{if(!i.running||!i.items.length)return;const n=i.items;let o,a=n.length-1,r=!1;for(;a>=0;--a)o=n[a],o._active?(o._total>i.duration&&(i.duration=o._total),o.tick(t),r=!0):(n[a]=n[n.length-1],n.pop());r&&(s.draw(),this._notify(s,i,t,"progress")),n.length||(i.running=!1,this._notify(s,i,t,"complete"),i.initial=!1),e+=n.length})),this._lastDate=t,0===e&&(this._running=!1)}_getAnims(t){const e=this._charts;let i=e.get(t);return i||(i={running:!1,initial:!0,items:[],listeners:{complete:[],progress:[]}},e.set(t,i)),i}listen(t,e,i){this._getAnims(t).listeners[e].push(i)}add(t,e){e&&e.length&&this._getAnims(t).items.push(...e)}has(t){return this._getAnims(t).items.length>0}start(t){const e=this._charts.get(t);e&&(e.running=!0,e.start=Date.now(),e.duration=e.items.reduce(((t,e)=>Math.max(t,e._duration)),0),this._refresh())}running(t){if(!this._running)return!1;const e=this._charts.get(t);return!!(e&&e.running&&e.items.length)}stop(t){const e=this._charts.get(t);if(!e||!e.items.length)return;const i=e.items;let s=i.length-1;for(;s>=0;--s)i[s].cancel();e.items=[],this._notify(t,e,Date.now(),"complete")}remove(t){return this._charts.delete(t)}}var xt=new bt; +/*! + * @kurkle/color v0.3.2 + * https://github.com/kurkle/color#readme + * (c) 2023 Jukka Kurkela + * Released under the MIT License + */function _t(t){return t+.5|0}const yt=(t,e,i)=>Math.max(Math.min(t,i),e);function vt(t){return yt(_t(2.55*t),0,255)}function Mt(t){return yt(_t(255*t),0,255)}function wt(t){return yt(_t(t/2.55)/100,0,1)}function kt(t){return yt(_t(100*t),0,100)}const St={0:0,1:1,2:2,3:3,4:4,5:5,6:6,7:7,8:8,9:9,A:10,B:11,C:12,D:13,E:14,F:15,a:10,b:11,c:12,d:13,e:14,f:15},Pt=[..."0123456789ABCDEF"],Dt=t=>Pt[15&t],Ct=t=>Pt[(240&t)>>4]+Pt[15&t],Ot=t=>(240&t)>>4==(15&t);function At(t){var e=(t=>Ot(t.r)&&Ot(t.g)&&Ot(t.b)&&Ot(t.a))(t)?Dt:Ct;return t?"#"+e(t.r)+e(t.g)+e(t.b)+((t,e)=>t<255?e(t):"")(t.a,e):void 0}const Tt=/^(hsla?|hwb|hsv)\(\s*([-+.e\d]+)(?:deg)?[\s,]+([-+.e\d]+)%[\s,]+([-+.e\d]+)%(?:[\s,]+([-+.e\d]+)(%)?)?\s*\)$/;function Lt(t,e,i){const s=e*Math.min(i,1-i),n=(e,n=(e+t/30)%12)=>i-s*Math.max(Math.min(n-3,9-n,1),-1);return[n(0),n(8),n(4)]}function Et(t,e,i){const s=(s,n=(s+t/60)%6)=>i-i*e*Math.max(Math.min(n,4-n,1),0);return[s(5),s(3),s(1)]}function Rt(t,e,i){const s=Lt(t,1,.5);let n;for(e+i>1&&(n=1/(e+i),e*=n,i*=n),n=0;n<3;n++)s[n]*=1-e-i,s[n]+=e;return s}function It(t){const e=t.r/255,i=t.g/255,s=t.b/255,n=Math.max(e,i,s),o=Math.min(e,i,s),a=(n+o)/2;let r,l,h;return n!==o&&(h=n-o,l=a>.5?h/(2-n-o):h/(n+o),r=function(t,e,i,s,n){return t===n?(e-i)/s+(e>16&255,o>>8&255,255&o]}return t}(),Ht.transparent=[0,0,0,0]);const e=Ht[t.toLowerCase()];return e&&{r:e[0],g:e[1],b:e[2],a:4===e.length?e[3]:255}}const $t=/^rgba?\(\s*([-+.\d]+)(%)?[\s,]+([-+.e\d]+)(%)?[\s,]+([-+.e\d]+)(%)?(?:[\s,/]+([-+.e\d]+)(%)?)?\s*\)$/;const Yt=t=>t<=.0031308?12.92*t:1.055*Math.pow(t,1/2.4)-.055,Ut=t=>t<=.04045?t/12.92:Math.pow((t+.055)/1.055,2.4);function Xt(t,e,i){if(t){let s=It(t);s[e]=Math.max(0,Math.min(s[e]+s[e]*i,0===e?360:1)),s=Ft(s),t.r=s[0],t.g=s[1],t.b=s[2]}}function qt(t,e){return t?Object.assign(e||{},t):t}function Kt(t){var e={r:0,g:0,b:0,a:255};return Array.isArray(t)?t.length>=3&&(e={r:t[0],g:t[1],b:t[2],a:255},t.length>3&&(e.a=Mt(t[3]))):(e=qt(t,{r:0,g:0,b:0,a:1})).a=Mt(e.a),e}function Gt(t){return"r"===t.charAt(0)?function(t){const e=$t.exec(t);let i,s,n,o=255;if(e){if(e[7]!==i){const t=+e[7];o=e[8]?vt(t):yt(255*t,0,255)}return i=+e[1],s=+e[3],n=+e[5],i=255&(e[2]?vt(i):yt(i,0,255)),s=255&(e[4]?vt(s):yt(s,0,255)),n=255&(e[6]?vt(n):yt(n,0,255)),{r:i,g:s,b:n,a:o}}}(t):Bt(t)}class Zt{constructor(t){if(t instanceof Zt)return t;const e=typeof t;let i;var s,n,o;"object"===e?i=Kt(t):"string"===e&&(o=(s=t).length,"#"===s[0]&&(4===o||5===o?n={r:255&17*St[s[1]],g:255&17*St[s[2]],b:255&17*St[s[3]],a:5===o?17*St[s[4]]:255}:7!==o&&9!==o||(n={r:St[s[1]]<<4|St[s[2]],g:St[s[3]]<<4|St[s[4]],b:St[s[5]]<<4|St[s[6]],a:9===o?St[s[7]]<<4|St[s[8]]:255})),i=n||jt(t)||Gt(t)),this._rgb=i,this._valid=!!i}get valid(){return this._valid}get rgb(){var t=qt(this._rgb);return t&&(t.a=wt(t.a)),t}set rgb(t){this._rgb=Kt(t)}rgbString(){return this._valid?(t=this._rgb)&&(t.a<255?`rgba(${t.r}, ${t.g}, ${t.b}, ${wt(t.a)})`:`rgb(${t.r}, ${t.g}, ${t.b})`):void 0;var t}hexString(){return this._valid?At(this._rgb):void 0}hslString(){return this._valid?function(t){if(!t)return;const e=It(t),i=e[0],s=kt(e[1]),n=kt(e[2]);return t.a<255?`hsla(${i}, ${s}%, ${n}%, ${wt(t.a)})`:`hsl(${i}, ${s}%, ${n}%)`}(this._rgb):void 0}mix(t,e){if(t){const i=this.rgb,s=t.rgb;let n;const o=e===n?.5:e,a=2*o-1,r=i.a-s.a,l=((a*r==-1?a:(a+r)/(1+a*r))+1)/2;n=1-l,i.r=255&l*i.r+n*s.r+.5,i.g=255&l*i.g+n*s.g+.5,i.b=255&l*i.b+n*s.b+.5,i.a=o*i.a+(1-o)*s.a,this.rgb=i}return this}interpolate(t,e){return t&&(this._rgb=function(t,e,i){const s=Ut(wt(t.r)),n=Ut(wt(t.g)),o=Ut(wt(t.b));return{r:Mt(Yt(s+i*(Ut(wt(e.r))-s))),g:Mt(Yt(n+i*(Ut(wt(e.g))-n))),b:Mt(Yt(o+i*(Ut(wt(e.b))-o))),a:t.a+i*(e.a-t.a)}}(this._rgb,t._rgb,e)),this}clone(){return new Zt(this.rgb)}alpha(t){return this._rgb.a=Mt(t),this}clearer(t){return this._rgb.a*=1-t,this}greyscale(){const t=this._rgb,e=_t(.3*t.r+.59*t.g+.11*t.b);return t.r=t.g=t.b=e,this}opaquer(t){return this._rgb.a*=1+t,this}negate(){const t=this._rgb;return t.r=255-t.r,t.g=255-t.g,t.b=255-t.b,this}lighten(t){return Xt(this._rgb,2,t),this}darken(t){return Xt(this._rgb,2,-t),this}saturate(t){return Xt(this._rgb,1,t),this}desaturate(t){return Xt(this._rgb,1,-t),this}rotate(t){return function(t,e){var i=It(t);i[0]=Vt(i[0]+e),i=Ft(i),t.r=i[0],t.g=i[1],t.b=i[2]}(this._rgb,t),this}}function Jt(t){if(t&&"object"==typeof t){const e=t.toString();return"[object CanvasPattern]"===e||"[object CanvasGradient]"===e}return!1}function Qt(t){return Jt(t)?t:new Zt(t)}function te(t){return Jt(t)?t:new Zt(t).saturate(.5).darken(.1).hexString()}const ee=["x","y","borderWidth","radius","tension"],ie=["color","borderColor","backgroundColor"];const se=new Map;function ne(t,e,i){return function(t,e){e=e||{};const i=t+JSON.stringify(e);let s=se.get(i);return s||(s=new Intl.NumberFormat(t,e),se.set(i,s)),s}(e,i).format(t)}const oe={values:t=>n(t)?t:""+t,numeric(t,e,i){if(0===t)return"0";const s=this.chart.options.locale;let n,o=t;if(i.length>1){const e=Math.max(Math.abs(i[0].value),Math.abs(i[i.length-1].value));(e<1e-4||e>1e15)&&(n="scientific"),o=function(t,e){let i=e.length>3?e[2].value-e[1].value:e[1].value-e[0].value;Math.abs(i)>=1&&t!==Math.floor(t)&&(i=t-Math.floor(t));return i}(t,i)}const a=z(Math.abs(o)),r=isNaN(a)?1:Math.max(Math.min(-1*Math.floor(a),20),0),l={notation:n,minimumFractionDigits:r,maximumFractionDigits:r};return Object.assign(l,this.options.ticks.format),ne(t,s,l)},logarithmic(t,e,i){if(0===t)return"0";const s=i[e].significand||t/Math.pow(10,Math.floor(z(t)));return[1,2,3,5,10,15].includes(s)||e>.8*i.length?oe.numeric.call(this,t,e,i):""}};var ae={formatters:oe};const re=Object.create(null),le=Object.create(null);function he(t,e){if(!e)return t;const i=e.split(".");for(let e=0,s=i.length;et.chart.platform.getDevicePixelRatio(),this.elements={},this.events=["mousemove","mouseout","click","touchstart","touchmove"],this.font={family:"'Helvetica Neue', 'Helvetica', 'Arial', sans-serif",size:12,style:"normal",lineHeight:1.2,weight:null},this.hover={},this.hoverBackgroundColor=(t,e)=>te(e.backgroundColor),this.hoverBorderColor=(t,e)=>te(e.borderColor),this.hoverColor=(t,e)=>te(e.color),this.indexAxis="x",this.interaction={mode:"nearest",intersect:!0,includeInvisible:!1},this.maintainAspectRatio=!0,this.onHover=null,this.onClick=null,this.parsing=!0,this.plugins={},this.responsive=!0,this.scale=void 0,this.scales={},this.showLine=!0,this.drawActiveElementsOnTop=!0,this.describe(t),this.apply(e)}set(t,e){return ce(this,t,e)}get(t){return he(this,t)}describe(t,e){return ce(le,t,e)}override(t,e){return ce(re,t,e)}route(t,e,i,s){const n=he(this,t),a=he(this,i),r="_"+e;Object.defineProperties(n,{[r]:{value:n[e],writable:!0},[e]:{enumerable:!0,get(){const t=this[r],e=a[s];return o(t)?Object.assign({},e,t):l(t,e)},set(t){this[r]=t}}})}apply(t){t.forEach((t=>t(this)))}}var ue=new de({_scriptable:t=>!t.startsWith("on"),_indexable:t=>"events"!==t,hover:{_fallback:"interaction"},interaction:{_scriptable:!1,_indexable:!1}},[function(t){t.set("animation",{delay:void 0,duration:1e3,easing:"easeOutQuart",fn:void 0,from:void 0,loop:void 0,to:void 0,type:void 0}),t.describe("animation",{_fallback:!1,_indexable:!1,_scriptable:t=>"onProgress"!==t&&"onComplete"!==t&&"fn"!==t}),t.set("animations",{colors:{type:"color",properties:ie},numbers:{type:"number",properties:ee}}),t.describe("animations",{_fallback:"animation"}),t.set("transitions",{active:{animation:{duration:400}},resize:{animation:{duration:0}},show:{animations:{colors:{from:"transparent"},visible:{type:"boolean",duration:0}}},hide:{animations:{colors:{to:"transparent"},visible:{type:"boolean",easing:"linear",fn:t=>0|t}}}})},function(t){t.set("layout",{autoPadding:!0,padding:{top:0,right:0,bottom:0,left:0}})},function(t){t.set("scale",{display:!0,offset:!1,reverse:!1,beginAtZero:!1,bounds:"ticks",grace:0,grid:{display:!0,lineWidth:1,drawOnChartArea:!0,drawTicks:!0,tickLength:8,tickWidth:(t,e)=>e.lineWidth,tickColor:(t,e)=>e.color,offset:!1},border:{display:!0,dash:[],dashOffset:0,width:1},title:{display:!1,text:"",padding:{top:4,bottom:4}},ticks:{minRotation:0,maxRotation:50,mirror:!1,textStrokeWidth:0,textStrokeColor:"",padding:3,display:!0,autoSkip:!0,autoSkipPadding:3,labelOffset:0,callback:ae.formatters.values,minor:{},major:{},align:"center",crossAlign:"near",showLabelBackdrop:!1,backdropColor:"rgba(255, 255, 255, 0.75)",backdropPadding:2}}),t.route("scale.ticks","color","","color"),t.route("scale.grid","color","","borderColor"),t.route("scale.border","color","","borderColor"),t.route("scale.title","color","","color"),t.describe("scale",{_fallback:!1,_scriptable:t=>!t.startsWith("before")&&!t.startsWith("after")&&"callback"!==t&&"parser"!==t,_indexable:t=>"borderDash"!==t&&"tickBorderDash"!==t&&"dash"!==t}),t.describe("scales",{_fallback:"scale"}),t.describe("scale.ticks",{_scriptable:t=>"backdropPadding"!==t&&"callback"!==t,_indexable:t=>"backdropPadding"!==t})}]);function fe(){return"undefined"!=typeof window&&"undefined"!=typeof document}function ge(t){let e=t.parentNode;return e&&"[object ShadowRoot]"===e.toString()&&(e=e.host),e}function pe(t,e,i){let s;return"string"==typeof t?(s=parseInt(t,10),-1!==t.indexOf("%")&&(s=s/100*e.parentNode[i])):s=t,s}const me=t=>t.ownerDocument.defaultView.getComputedStyle(t,null);function be(t,e){return me(t).getPropertyValue(e)}const xe=["top","right","bottom","left"];function _e(t,e,i){const s={};i=i?"-"+i:"";for(let n=0;n<4;n++){const o=xe[n];s[o]=parseFloat(t[e+"-"+o+i])||0}return s.width=s.left+s.right,s.height=s.top+s.bottom,s}const ye=(t,e,i)=>(t>0||e>0)&&(!i||!i.shadowRoot);function ve(t,e){if("native"in t)return t;const{canvas:i,currentDevicePixelRatio:s}=e,n=me(i),o="border-box"===n.boxSizing,a=_e(n,"padding"),r=_e(n,"border","width"),{x:l,y:h,box:c}=function(t,e){const i=t.touches,s=i&&i.length?i[0]:t,{offsetX:n,offsetY:o}=s;let a,r,l=!1;if(ye(n,o,t.target))a=n,r=o;else{const t=e.getBoundingClientRect();a=s.clientX-t.left,r=s.clientY-t.top,l=!0}return{x:a,y:r,box:l}}(t,i),d=a.left+(c&&r.left),u=a.top+(c&&r.top);let{width:f,height:g}=e;return o&&(f-=a.width+r.width,g-=a.height+r.height),{x:Math.round((l-d)/f*i.width/s),y:Math.round((h-u)/g*i.height/s)}}const Me=t=>Math.round(10*t)/10;function we(t,e,i,s){const n=me(t),o=_e(n,"margin"),a=pe(n.maxWidth,t,"clientWidth")||T,r=pe(n.maxHeight,t,"clientHeight")||T,l=function(t,e,i){let s,n;if(void 0===e||void 0===i){const o=ge(t);if(o){const t=o.getBoundingClientRect(),a=me(o),r=_e(a,"border","width"),l=_e(a,"padding");e=t.width-l.width-r.width,i=t.height-l.height-r.height,s=pe(a.maxWidth,o,"clientWidth"),n=pe(a.maxHeight,o,"clientHeight")}else e=t.clientWidth,i=t.clientHeight}return{width:e,height:i,maxWidth:s||T,maxHeight:n||T}}(t,e,i);let{width:h,height:c}=l;if("content-box"===n.boxSizing){const t=_e(n,"border","width"),e=_e(n,"padding");h-=e.width+t.width,c-=e.height+t.height}h=Math.max(0,h-o.width),c=Math.max(0,s?h/s:c-o.height),h=Me(Math.min(h,a,l.maxWidth)),c=Me(Math.min(c,r,l.maxHeight)),h&&!c&&(c=Me(h/2));return(void 0!==e||void 0!==i)&&s&&l.height&&c>l.height&&(c=l.height,h=Me(Math.floor(c*s))),{width:h,height:c}}function ke(t,e,i){const s=e||1,n=Math.floor(t.height*s),o=Math.floor(t.width*s);t.height=Math.floor(t.height),t.width=Math.floor(t.width);const a=t.canvas;return a.style&&(i||!a.style.height&&!a.style.width)&&(a.style.height=`${t.height}px`,a.style.width=`${t.width}px`),(t.currentDevicePixelRatio!==s||a.height!==n||a.width!==o)&&(t.currentDevicePixelRatio=s,a.height=n,a.width=o,t.ctx.setTransform(s,0,0,s,0,0),!0)}const Se=function(){let t=!1;try{const e={get passive(){return t=!0,!1}};window.addEventListener("test",null,e),window.removeEventListener("test",null,e)}catch(t){}return t}();function Pe(t,e){const i=be(t,e),s=i&&i.match(/^(\d+)(\.\d+)?px$/);return s?+s[1]:void 0}function De(t){return!t||s(t.size)||s(t.family)?null:(t.style?t.style+" ":"")+(t.weight?t.weight+" ":"")+t.size+"px "+t.family}function Ce(t,e,i,s,n){let o=e[n];return o||(o=e[n]=t.measureText(n).width,i.push(n)),o>s&&(s=o),s}function Oe(t,e,i,s){let o=(s=s||{}).data=s.data||{},a=s.garbageCollect=s.garbageCollect||[];s.font!==e&&(o=s.data={},a=s.garbageCollect=[],s.font=e),t.save(),t.font=e;let r=0;const l=i.length;let h,c,d,u,f;for(h=0;hi.length){for(h=0;h0&&t.stroke()}}function Re(t,e,i){return i=i||.5,!e||t&&t.x>e.left-i&&t.xe.top-i&&t.y0&&""!==r.strokeColor;let c,d;for(t.save(),t.font=a.string,function(t,e){e.translation&&t.translate(e.translation[0],e.translation[1]),s(e.rotation)||t.rotate(e.rotation),e.color&&(t.fillStyle=e.color),e.textAlign&&(t.textAlign=e.textAlign),e.textBaseline&&(t.textBaseline=e.textBaseline)}(t,r),c=0;ct[0])){const o=i||t;void 0===s&&(s=ti("_fallback",t));const a={[Symbol.toStringTag]:"Object",_cacheable:!0,_scopes:t,_rootScopes:o,_fallback:s,_getTarget:n,override:i=>je([i,...t],e,o,s)};return new Proxy(a,{deleteProperty:(e,i)=>(delete e[i],delete e._keys,delete t[0][i],!0),get:(i,s)=>qe(i,s,(()=>function(t,e,i,s){let n;for(const o of e)if(n=ti(Ue(o,t),i),void 0!==n)return Xe(t,n)?Je(i,s,t,n):n}(s,e,t,i))),getOwnPropertyDescriptor:(t,e)=>Reflect.getOwnPropertyDescriptor(t._scopes[0],e),getPrototypeOf:()=>Reflect.getPrototypeOf(t[0]),has:(t,e)=>ei(t).includes(e),ownKeys:t=>ei(t),set(t,e,i){const s=t._storage||(t._storage=n());return t[e]=s[e]=i,delete t._keys,!0}})}function $e(t,e,i,s){const a={_cacheable:!1,_proxy:t,_context:e,_subProxy:i,_stack:new Set,_descriptors:Ye(t,s),setContext:e=>$e(t,e,i,s),override:n=>$e(t.override(n),e,i,s)};return new Proxy(a,{deleteProperty:(e,i)=>(delete e[i],delete t[i],!0),get:(t,e,i)=>qe(t,e,(()=>function(t,e,i){const{_proxy:s,_context:a,_subProxy:r,_descriptors:l}=t;let h=s[e];S(h)&&l.isScriptable(e)&&(h=function(t,e,i,s){const{_proxy:n,_context:o,_subProxy:a,_stack:r}=i;if(r.has(t))throw new Error("Recursion detected: "+Array.from(r).join("->")+"->"+t);r.add(t);let l=e(o,a||s);r.delete(t),Xe(t,l)&&(l=Je(n._scopes,n,t,l));return l}(e,h,t,i));n(h)&&h.length&&(h=function(t,e,i,s){const{_proxy:n,_context:a,_subProxy:r,_descriptors:l}=i;if(void 0!==a.index&&s(t))return e[a.index%e.length];if(o(e[0])){const i=e,s=n._scopes.filter((t=>t!==i));e=[];for(const o of i){const i=Je(s,n,t,o);e.push($e(i,a,r&&r[t],l))}}return e}(e,h,t,l.isIndexable));Xe(e,h)&&(h=$e(h,a,r&&r[e],l));return h}(t,e,i))),getOwnPropertyDescriptor:(e,i)=>e._descriptors.allKeys?Reflect.has(t,i)?{enumerable:!0,configurable:!0}:void 0:Reflect.getOwnPropertyDescriptor(t,i),getPrototypeOf:()=>Reflect.getPrototypeOf(t),has:(e,i)=>Reflect.has(t,i),ownKeys:()=>Reflect.ownKeys(t),set:(e,i,s)=>(t[i]=s,delete e[i],!0)})}function Ye(t,e={scriptable:!0,indexable:!0}){const{_scriptable:i=e.scriptable,_indexable:s=e.indexable,_allKeys:n=e.allKeys}=t;return{allKeys:n,scriptable:i,indexable:s,isScriptable:S(i)?i:()=>i,isIndexable:S(s)?s:()=>s}}const Ue=(t,e)=>t?t+w(e):e,Xe=(t,e)=>o(e)&&"adapters"!==t&&(null===Object.getPrototypeOf(e)||e.constructor===Object);function qe(t,e,i){if(Object.prototype.hasOwnProperty.call(t,e))return t[e];const s=i();return t[e]=s,s}function Ke(t,e,i){return S(t)?t(e,i):t}const Ge=(t,e)=>!0===t?e:"string"==typeof t?M(e,t):void 0;function Ze(t,e,i,s,n){for(const o of e){const e=Ge(i,o);if(e){t.add(e);const o=Ke(e._fallback,i,n);if(void 0!==o&&o!==i&&o!==s)return o}else if(!1===e&&void 0!==s&&i!==s)return null}return!1}function Je(t,e,i,s){const a=e._rootScopes,r=Ke(e._fallback,i,s),l=[...t,...a],h=new Set;h.add(s);let c=Qe(h,l,i,r||i,s);return null!==c&&((void 0===r||r===i||(c=Qe(h,l,r,c,s),null!==c))&&je(Array.from(h),[""],a,r,(()=>function(t,e,i){const s=t._getTarget();e in s||(s[e]={});const a=s[e];if(n(a)&&o(i))return i;return a||{}}(e,i,s))))}function Qe(t,e,i,s,n){for(;i;)i=Ze(t,e,i,s,n);return i}function ti(t,e){for(const i of e){if(!i)continue;const e=i[t];if(void 0!==e)return e}}function ei(t){let e=t._keys;return e||(e=t._keys=function(t){const e=new Set;for(const i of t)for(const t of Object.keys(i).filter((t=>!t.startsWith("_"))))e.add(t);return Array.from(e)}(t._scopes)),e}function ii(t,e,i,s){const{iScale:n}=t,{key:o="r"}=this._parsing,a=new Array(s);let r,l,h,c;for(r=0,l=s;re"x"===t?"y":"x";function ai(t,e,i,s){const n=t.skip?e:t,o=e,a=i.skip?e:i,r=q(o,n),l=q(a,o);let h=r/(r+l),c=l/(r+l);h=isNaN(h)?0:h,c=isNaN(c)?0:c;const d=s*h,u=s*c;return{previous:{x:o.x-d*(a.x-n.x),y:o.y-d*(a.y-n.y)},next:{x:o.x+u*(a.x-n.x),y:o.y+u*(a.y-n.y)}}}function ri(t,e="x"){const i=oi(e),s=t.length,n=Array(s).fill(0),o=Array(s);let a,r,l,h=ni(t,0);for(a=0;a!t.skip))),"monotone"===e.cubicInterpolationMode)ri(t,n);else{let i=s?t[t.length-1]:t[0];for(o=0,a=t.length;o0===t||1===t,di=(t,e,i)=>-Math.pow(2,10*(t-=1))*Math.sin((t-e)*O/i),ui=(t,e,i)=>Math.pow(2,-10*t)*Math.sin((t-e)*O/i)+1,fi={linear:t=>t,easeInQuad:t=>t*t,easeOutQuad:t=>-t*(t-2),easeInOutQuad:t=>(t/=.5)<1?.5*t*t:-.5*(--t*(t-2)-1),easeInCubic:t=>t*t*t,easeOutCubic:t=>(t-=1)*t*t+1,easeInOutCubic:t=>(t/=.5)<1?.5*t*t*t:.5*((t-=2)*t*t+2),easeInQuart:t=>t*t*t*t,easeOutQuart:t=>-((t-=1)*t*t*t-1),easeInOutQuart:t=>(t/=.5)<1?.5*t*t*t*t:-.5*((t-=2)*t*t*t-2),easeInQuint:t=>t*t*t*t*t,easeOutQuint:t=>(t-=1)*t*t*t*t+1,easeInOutQuint:t=>(t/=.5)<1?.5*t*t*t*t*t:.5*((t-=2)*t*t*t*t+2),easeInSine:t=>1-Math.cos(t*E),easeOutSine:t=>Math.sin(t*E),easeInOutSine:t=>-.5*(Math.cos(C*t)-1),easeInExpo:t=>0===t?0:Math.pow(2,10*(t-1)),easeOutExpo:t=>1===t?1:1-Math.pow(2,-10*t),easeInOutExpo:t=>ci(t)?t:t<.5?.5*Math.pow(2,10*(2*t-1)):.5*(2-Math.pow(2,-10*(2*t-1))),easeInCirc:t=>t>=1?t:-(Math.sqrt(1-t*t)-1),easeOutCirc:t=>Math.sqrt(1-(t-=1)*t),easeInOutCirc:t=>(t/=.5)<1?-.5*(Math.sqrt(1-t*t)-1):.5*(Math.sqrt(1-(t-=2)*t)+1),easeInElastic:t=>ci(t)?t:di(t,.075,.3),easeOutElastic:t=>ci(t)?t:ui(t,.075,.3),easeInOutElastic(t){const e=.1125;return ci(t)?t:t<.5?.5*di(2*t,e,.45):.5+.5*ui(2*t-1,e,.45)},easeInBack(t){const e=1.70158;return t*t*((e+1)*t-e)},easeOutBack(t){const e=1.70158;return(t-=1)*t*((e+1)*t+e)+1},easeInOutBack(t){let e=1.70158;return(t/=.5)<1?t*t*((1+(e*=1.525))*t-e)*.5:.5*((t-=2)*t*((1+(e*=1.525))*t+e)+2)},easeInBounce:t=>1-fi.easeOutBounce(1-t),easeOutBounce(t){const e=7.5625,i=2.75;return t<1/i?e*t*t:t<2/i?e*(t-=1.5/i)*t+.75:t<2.5/i?e*(t-=2.25/i)*t+.9375:e*(t-=2.625/i)*t+.984375},easeInOutBounce:t=>t<.5?.5*fi.easeInBounce(2*t):.5*fi.easeOutBounce(2*t-1)+.5};function gi(t,e,i,s){return{x:t.x+i*(e.x-t.x),y:t.y+i*(e.y-t.y)}}function pi(t,e,i,s){return{x:t.x+i*(e.x-t.x),y:"middle"===s?i<.5?t.y:e.y:"after"===s?i<1?t.y:e.y:i>0?e.y:t.y}}function mi(t,e,i,s){const n={x:t.cp2x,y:t.cp2y},o={x:e.cp1x,y:e.cp1y},a=gi(t,n,i),r=gi(n,o,i),l=gi(o,e,i),h=gi(a,r,i),c=gi(r,l,i);return gi(h,c,i)}const bi=/^(normal|(\d+(?:\.\d+)?)(px|em|%)?)$/,xi=/^(normal|italic|initial|inherit|unset|(oblique( -?[0-9]?[0-9]deg)?))$/;function _i(t,e){const i=(""+t).match(bi);if(!i||"normal"===i[1])return 1.2*e;switch(t=+i[2],i[3]){case"px":return t;case"%":t/=100}return e*t}const yi=t=>+t||0;function vi(t,e){const i={},s=o(e),n=s?Object.keys(e):e,a=o(t)?s?i=>l(t[i],t[e[i]]):e=>t[e]:()=>t;for(const t of n)i[t]=yi(a(t));return i}function Mi(t){return vi(t,{top:"y",right:"x",bottom:"y",left:"x"})}function wi(t){return vi(t,["topLeft","topRight","bottomLeft","bottomRight"])}function ki(t){const e=Mi(t);return e.width=e.left+e.right,e.height=e.top+e.bottom,e}function Si(t,e){t=t||{},e=e||ue.font;let i=l(t.size,e.size);"string"==typeof i&&(i=parseInt(i,10));let s=l(t.style,e.style);s&&!(""+s).match(xi)&&(console.warn('Invalid font style specified: "'+s+'"'),s=void 0);const n={family:l(t.family,e.family),lineHeight:_i(l(t.lineHeight,e.lineHeight),i),size:i,style:s,weight:l(t.weight,e.weight),string:""};return n.string=De(n),n}function Pi(t,e,i,s){let o,a,r,l=!0;for(o=0,a=t.length;oi&&0===t?0:t+e;return{min:a(s,-Math.abs(o)),max:a(n,o)}}function Ci(t,e){return Object.assign(Object.create(t),e)}function Oi(t,e,i){return t?function(t,e){return{x:i=>t+t+e-i,setWidth(t){e=t},textAlign:t=>"center"===t?t:"right"===t?"left":"right",xPlus:(t,e)=>t-e,leftForLtr:(t,e)=>t-e}}(e,i):{x:t=>t,setWidth(t){},textAlign:t=>t,xPlus:(t,e)=>t+e,leftForLtr:(t,e)=>t}}function Ai(t,e){let i,s;"ltr"!==e&&"rtl"!==e||(i=t.canvas.style,s=[i.getPropertyValue("direction"),i.getPropertyPriority("direction")],i.setProperty("direction",e,"important"),t.prevTextDirection=s)}function Ti(t,e){void 0!==e&&(delete t.prevTextDirection,t.canvas.style.setProperty("direction",e[0],e[1]))}function Li(t){return"angle"===t?{between:Z,compare:K,normalize:G}:{between:tt,compare:(t,e)=>t-e,normalize:t=>t}}function Ei({start:t,end:e,count:i,loop:s,style:n}){return{start:t%i,end:e%i,loop:s&&(e-t+1)%i==0,style:n}}function Ri(t,e,i){if(!i)return[t];const{property:s,start:n,end:o}=i,a=e.length,{compare:r,between:l,normalize:h}=Li(s),{start:c,end:d,loop:u,style:f}=function(t,e,i){const{property:s,start:n,end:o}=i,{between:a,normalize:r}=Li(s),l=e.length;let h,c,{start:d,end:u,loop:f}=t;if(f){for(d+=l,u+=l,h=0,c=l;hx||l(n,b,p)&&0!==r(n,b),v=()=>!x||0===r(o,p)||l(o,b,p);for(let t=c,i=c;t<=d;++t)m=e[t%a],m.skip||(p=h(m[s]),p!==b&&(x=l(p,n,o),null===_&&y()&&(_=0===r(p,n)?t:i),null!==_&&v()&&(g.push(Ei({start:_,end:t,loop:u,count:a,style:f})),_=null),i=t,b=p));return null!==_&&g.push(Ei({start:_,end:d,loop:u,count:a,style:f})),g}function Ii(t,e){const i=[],s=t.segments;for(let n=0;nn&&t[o%e].skip;)o--;return o%=e,{start:n,end:o}}(i,n,o,s);if(!0===s)return Fi(t,[{start:a,end:r,loop:o}],i,e);return Fi(t,function(t,e,i,s){const n=t.length,o=[];let a,r=e,l=t[e];for(a=e+1;a<=i;++a){const i=t[a%n];i.skip||i.stop?l.skip||(s=!1,o.push({start:e%n,end:(a-1)%n,loop:s}),e=r=i.stop?a:null):(r=a,l.skip&&(e=a)),l=i}return null!==r&&o.push({start:e%n,end:r%n,loop:s}),o}(i,a,r{t[a](e[i],n)&&(o.push({element:t,datasetIndex:s,index:l}),r=r||t.inRange(e.x,e.y,n))})),s&&!r?[]:o}var Xi={evaluateInteractionItems:Hi,modes:{index(t,e,i,s){const n=ve(e,t),o=i.axis||"x",a=i.includeInvisible||!1,r=i.intersect?ji(t,n,o,s,a):Yi(t,n,o,!1,s,a),l=[];return r.length?(t.getSortedVisibleDatasetMetas().forEach((t=>{const e=r[0].index,i=t.data[e];i&&!i.skip&&l.push({element:i,datasetIndex:t.index,index:e})})),l):[]},dataset(t,e,i,s){const n=ve(e,t),o=i.axis||"xy",a=i.includeInvisible||!1;let r=i.intersect?ji(t,n,o,s,a):Yi(t,n,o,!1,s,a);if(r.length>0){const e=r[0].datasetIndex,i=t.getDatasetMeta(e).data;r=[];for(let t=0;tji(t,ve(e,t),i.axis||"xy",s,i.includeInvisible||!1),nearest(t,e,i,s){const n=ve(e,t),o=i.axis||"xy",a=i.includeInvisible||!1;return Yi(t,n,o,i.intersect,s,a)},x:(t,e,i,s)=>Ui(t,ve(e,t),"x",i.intersect,s),y:(t,e,i,s)=>Ui(t,ve(e,t),"y",i.intersect,s)}};const qi=["left","top","right","bottom"];function Ki(t,e){return t.filter((t=>t.pos===e))}function Gi(t,e){return t.filter((t=>-1===qi.indexOf(t.pos)&&t.box.axis===e))}function Zi(t,e){return t.sort(((t,i)=>{const s=e?i:t,n=e?t:i;return s.weight===n.weight?s.index-n.index:s.weight-n.weight}))}function Ji(t,e){const i=function(t){const e={};for(const i of t){const{stack:t,pos:s,stackWeight:n}=i;if(!t||!qi.includes(s))continue;const o=e[t]||(e[t]={count:0,placed:0,weight:0,size:0});o.count++,o.weight+=n}return e}(t),{vBoxMaxWidth:s,hBoxMaxHeight:n}=e;let o,a,r;for(o=0,a=t.length;o{s[t]=Math.max(e[t],i[t])})),s}return s(t?["left","right"]:["top","bottom"])}function ss(t,e,i,s){const n=[];let o,a,r,l,h,c;for(o=0,a=t.length,h=0;ot.box.fullSize)),!0),s=Zi(Ki(e,"left"),!0),n=Zi(Ki(e,"right")),o=Zi(Ki(e,"top"),!0),a=Zi(Ki(e,"bottom")),r=Gi(e,"x"),l=Gi(e,"y");return{fullSize:i,leftAndTop:s.concat(o),rightAndBottom:n.concat(l).concat(a).concat(r),chartArea:Ki(e,"chartArea"),vertical:s.concat(n).concat(l),horizontal:o.concat(a).concat(r)}}(t.boxes),l=r.vertical,h=r.horizontal;u(t.boxes,(t=>{"function"==typeof t.beforeLayout&&t.beforeLayout()}));const c=l.reduce(((t,e)=>e.box.options&&!1===e.box.options.display?t:t+1),0)||1,d=Object.freeze({outerWidth:e,outerHeight:i,padding:n,availableWidth:o,availableHeight:a,vBoxMaxWidth:o/2/c,hBoxMaxHeight:a/2}),f=Object.assign({},n);ts(f,ki(s));const g=Object.assign({maxPadding:f,w:o,h:a,x:n.left,y:n.top},n),p=Ji(l.concat(h),d);ss(r.fullSize,g,d,p),ss(l,g,d,p),ss(h,g,d,p)&&ss(l,g,d,p),function(t){const e=t.maxPadding;function i(i){const s=Math.max(e[i]-t[i],0);return t[i]+=s,s}t.y+=i("top"),t.x+=i("left"),i("right"),i("bottom")}(g),os(r.leftAndTop,g,d,p),g.x+=g.w,g.y+=g.h,os(r.rightAndBottom,g,d,p),t.chartArea={left:g.left,top:g.top,right:g.left+g.w,bottom:g.top+g.h,height:g.h,width:g.w},u(r.chartArea,(e=>{const i=e.box;Object.assign(i,t.chartArea),i.update(g.w,g.h,{left:0,top:0,right:0,bottom:0})}))}};class rs{acquireContext(t,e){}releaseContext(t){return!1}addEventListener(t,e,i){}removeEventListener(t,e,i){}getDevicePixelRatio(){return 1}getMaximumSize(t,e,i,s){return e=Math.max(0,e||t.width),i=i||t.height,{width:e,height:Math.max(0,s?Math.floor(e/s):i)}}isAttached(t){return!0}updateConfig(t){}}class ls extends rs{acquireContext(t){return t&&t.getContext&&t.getContext("2d")||null}updateConfig(t){t.options.animation=!1}}const hs="$chartjs",cs={touchstart:"mousedown",touchmove:"mousemove",touchend:"mouseup",pointerenter:"mouseenter",pointerdown:"mousedown",pointermove:"mousemove",pointerup:"mouseup",pointerleave:"mouseout",pointerout:"mouseout"},ds=t=>null===t||""===t;const us=!!Se&&{passive:!0};function fs(t,e,i){t.canvas.removeEventListener(e,i,us)}function gs(t,e){for(const i of t)if(i===e||i.contains(e))return!0}function ps(t,e,i){const s=t.canvas,n=new MutationObserver((t=>{let e=!1;for(const i of t)e=e||gs(i.addedNodes,s),e=e&&!gs(i.removedNodes,s);e&&i()}));return n.observe(document,{childList:!0,subtree:!0}),n}function ms(t,e,i){const s=t.canvas,n=new MutationObserver((t=>{let e=!1;for(const i of t)e=e||gs(i.removedNodes,s),e=e&&!gs(i.addedNodes,s);e&&i()}));return n.observe(document,{childList:!0,subtree:!0}),n}const bs=new Map;let xs=0;function _s(){const t=window.devicePixelRatio;t!==xs&&(xs=t,bs.forEach(((e,i)=>{i.currentDevicePixelRatio!==t&&e()})))}function ys(t,e,i){const s=t.canvas,n=s&&ge(s);if(!n)return;const o=ct(((t,e)=>{const s=n.clientWidth;i(t,e),s{const e=t[0],i=e.contentRect.width,s=e.contentRect.height;0===i&&0===s||o(i,s)}));return a.observe(n),function(t,e){bs.size||window.addEventListener("resize",_s),bs.set(t,e)}(t,o),a}function vs(t,e,i){i&&i.disconnect(),"resize"===e&&function(t){bs.delete(t),bs.size||window.removeEventListener("resize",_s)}(t)}function Ms(t,e,i){const s=t.canvas,n=ct((e=>{null!==t.ctx&&i(function(t,e){const i=cs[t.type]||t.type,{x:s,y:n}=ve(t,e);return{type:i,chart:e,native:t,x:void 0!==s?s:null,y:void 0!==n?n:null}}(e,t))}),t);return function(t,e,i){t.addEventListener(e,i,us)}(s,e,n),n}class ws extends rs{acquireContext(t,e){const i=t&&t.getContext&&t.getContext("2d");return i&&i.canvas===t?(function(t,e){const i=t.style,s=t.getAttribute("height"),n=t.getAttribute("width");if(t[hs]={initial:{height:s,width:n,style:{display:i.display,height:i.height,width:i.width}}},i.display=i.display||"block",i.boxSizing=i.boxSizing||"border-box",ds(n)){const e=Pe(t,"width");void 0!==e&&(t.width=e)}if(ds(s))if(""===t.style.height)t.height=t.width/(e||2);else{const e=Pe(t,"height");void 0!==e&&(t.height=e)}}(t,e),i):null}releaseContext(t){const e=t.canvas;if(!e[hs])return!1;const i=e[hs].initial;["height","width"].forEach((t=>{const n=i[t];s(n)?e.removeAttribute(t):e.setAttribute(t,n)}));const n=i.style||{};return Object.keys(n).forEach((t=>{e.style[t]=n[t]})),e.width=e.width,delete e[hs],!0}addEventListener(t,e,i){this.removeEventListener(t,e);const s=t.$proxies||(t.$proxies={}),n={attach:ps,detach:ms,resize:ys}[e]||Ms;s[e]=n(t,e,i)}removeEventListener(t,e){const i=t.$proxies||(t.$proxies={}),s=i[e];if(!s)return;({attach:vs,detach:vs,resize:vs}[e]||fs)(t,e,s),i[e]=void 0}getDevicePixelRatio(){return window.devicePixelRatio}getMaximumSize(t,e,i,s){return we(t,e,i,s)}isAttached(t){const e=ge(t);return!(!e||!e.isConnected)}}function ks(t){return!fe()||"undefined"!=typeof OffscreenCanvas&&t instanceof OffscreenCanvas?ls:ws}var Ss=Object.freeze({__proto__:null,BasePlatform:rs,BasicPlatform:ls,DomPlatform:ws,_detectPlatform:ks});const Ps="transparent",Ds={boolean:(t,e,i)=>i>.5?e:t,color(t,e,i){const s=Qt(t||Ps),n=s.valid&&Qt(e||Ps);return n&&n.valid?n.mix(s,i).hexString():e},number:(t,e,i)=>t+(e-t)*i};class Cs{constructor(t,e,i,s){const n=e[i];s=Pi([t.to,s,n,t.from]);const o=Pi([t.from,n,s]);this._active=!0,this._fn=t.fn||Ds[t.type||typeof o],this._easing=fi[t.easing]||fi.linear,this._start=Math.floor(Date.now()+(t.delay||0)),this._duration=this._total=Math.floor(t.duration),this._loop=!!t.loop,this._target=e,this._prop=i,this._from=o,this._to=s,this._promises=void 0}active(){return this._active}update(t,e,i){if(this._active){this._notify(!1);const s=this._target[this._prop],n=i-this._start,o=this._duration-n;this._start=i,this._duration=Math.floor(Math.max(o,t.duration)),this._total+=n,this._loop=!!t.loop,this._to=Pi([t.to,e,s,t.from]),this._from=Pi([t.from,s,e])}}cancel(){this._active&&(this.tick(Date.now()),this._active=!1,this._notify(!1))}tick(t){const e=t-this._start,i=this._duration,s=this._prop,n=this._from,o=this._loop,a=this._to;let r;if(this._active=n!==a&&(o||e1?2-r:r,r=this._easing(Math.min(1,Math.max(0,r))),this._target[s]=this._fn(n,a,r))}wait(){const t=this._promises||(this._promises=[]);return new Promise(((e,i)=>{t.push({res:e,rej:i})}))}_notify(t){const e=t?"res":"rej",i=this._promises||[];for(let t=0;t{const a=t[s];if(!o(a))return;const r={};for(const t of e)r[t]=a[t];(n(a.properties)&&a.properties||[s]).forEach((t=>{t!==s&&i.has(t)||i.set(t,r)}))}))}_animateOptions(t,e){const i=e.options,s=function(t,e){if(!e)return;let i=t.options;if(!i)return void(t.options=e);i.$shared&&(t.options=i=Object.assign({},i,{$shared:!1,$animations:{}}));return i}(t,i);if(!s)return[];const n=this._createAnimations(s,i);return i.$shared&&function(t,e){const i=[],s=Object.keys(e);for(let e=0;e{t.options=i}),(()=>{})),n}_createAnimations(t,e){const i=this._properties,s=[],n=t.$animations||(t.$animations={}),o=Object.keys(e),a=Date.now();let r;for(r=o.length-1;r>=0;--r){const l=o[r];if("$"===l.charAt(0))continue;if("options"===l){s.push(...this._animateOptions(t,e));continue}const h=e[l];let c=n[l];const d=i.get(l);if(c){if(d&&c.active()){c.update(d,h,a);continue}c.cancel()}d&&d.duration?(n[l]=c=new Cs(d,t,l,h),s.push(c)):t[l]=h}return s}update(t,e){if(0===this._properties.size)return void Object.assign(t,e);const i=this._createAnimations(t,e);return i.length?(xt.add(this._chart,i),!0):void 0}}function As(t,e){const i=t&&t.options||{},s=i.reverse,n=void 0===i.min?e:0,o=void 0===i.max?e:0;return{start:s?o:n,end:s?n:o}}function Ts(t,e){const i=[],s=t._getSortedDatasetMetas(e);let n,o;for(n=0,o=s.length;n0||!i&&e<0)return n.index}return null}function zs(t,e){const{chart:i,_cachedMeta:s}=t,n=i._stacks||(i._stacks={}),{iScale:o,vScale:a,index:r}=s,l=o.axis,h=a.axis,c=function(t,e,i){return`${t.id}.${e.id}.${i.stack||i.type}`}(o,a,s),d=e.length;let u;for(let t=0;ti[t].axis===e)).shift()}function Vs(t,e){const i=t.controller.index,s=t.vScale&&t.vScale.axis;if(s){e=e||t._parsed;for(const t of e){const e=t._stacks;if(!e||void 0===e[s]||void 0===e[s][i])return;delete e[s][i],void 0!==e[s]._visualValues&&void 0!==e[s]._visualValues[i]&&delete e[s]._visualValues[i]}}}const Bs=t=>"reset"===t||"none"===t,Ns=(t,e)=>e?t:Object.assign({},t);class Ws{static defaults={};static datasetElementType=null;static dataElementType=null;constructor(t,e){this.chart=t,this._ctx=t.ctx,this.index=e,this._cachedDataOpts={},this._cachedMeta=this.getMeta(),this._type=this._cachedMeta.type,this.options=void 0,this._parsing=!1,this._data=void 0,this._objectData=void 0,this._sharedOptions=void 0,this._drawStart=void 0,this._drawCount=void 0,this.enableOptionSharing=!1,this.supportsDecimation=!1,this.$context=void 0,this._syncList=[],this.datasetElementType=new.target.datasetElementType,this.dataElementType=new.target.dataElementType,this.initialize()}initialize(){const t=this._cachedMeta;this.configure(),this.linkScales(),t._stacked=Es(t.vScale,t),this.addElements(),this.options.fill&&!this.chart.isPluginEnabled("filler")&&console.warn("Tried to use the 'fill' option without the 'Filler' plugin enabled. Please import and register the 'Filler' plugin and make sure it is not disabled_ in the options")}updateIndex(t){this.index!==t&&Vs(this._cachedMeta),this.index=t}linkScales(){const t=this.chart,e=this._cachedMeta,i=this.getDataset(),s=(t,e,i,s)=>"x"===t?e:"r"===t?s:i,n=e.xAxisID=l(i.xAxisID,Fs(t,"x")),o=e.yAxisID=l(i.yAxisID,Fs(t,"y")),a=e.rAxisID=l(i.rAxisID,Fs(t,"r")),r=e.indexAxis,h=e.iAxisID=s(r,n,o,a),c=e.vAxisID=s(r,o,n,a);e.xScale=this.getScaleForId(n),e.yScale=this.getScaleForId(o),e.rScale=this.getScaleForId(a),e.iScale=this.getScaleForId(h),e.vScale=this.getScaleForId(c)}getDataset(){return this.chart.data.datasets[this.index]}getMeta(){return this.chart.getDatasetMeta(this.index)}getScaleForId(t){return this.chart.scales[t]}_getOtherScale(t){const e=this._cachedMeta;return t===e.iScale?e.vScale:e.iScale}reset(){this._update("reset")}_destroy(){const t=this._cachedMeta;this._data&&rt(this._data,this),t._stacked&&Vs(t)}_dataCheck(){const t=this.getDataset(),e=t.data||(t.data=[]),i=this._data;if(o(e))this._data=function(t){const e=Object.keys(t),i=new Array(e.length);let s,n,o;for(s=0,n=e.length;s0&&i._parsed[t-1];if(!1===this._parsing)i._parsed=s,i._sorted=!0,d=s;else{d=n(s[t])?this.parseArrayData(i,s,t,e):o(s[t])?this.parseObjectData(i,s,t,e):this.parsePrimitiveData(i,s,t,e);const a=()=>null===c[l]||f&&c[l]t&&!e.hidden&&e._stacked&&{keys:Ts(i,!0),values:null})(e,i,this.chart),h={min:Number.POSITIVE_INFINITY,max:Number.NEGATIVE_INFINITY},{min:c,max:d}=function(t){const{min:e,max:i,minDefined:s,maxDefined:n}=t.getUserBounds();return{min:s?e:Number.NEGATIVE_INFINITY,max:n?i:Number.POSITIVE_INFINITY}}(r);let u,f;function g(){f=s[u];const e=f[r.axis];return!a(f[t.axis])||c>e||d=0;--u)if(!g()){this.updateRangeFromParsed(h,t,f,l);break}return h}getAllParsedValues(t){const e=this._cachedMeta._parsed,i=[];let s,n,o;for(s=0,n=e.length;s=0&&tthis.getContext(i,s,e)),c);return f.$shared&&(f.$shared=r,n[o]=Object.freeze(Ns(f,r))),f}_resolveAnimations(t,e,i){const s=this.chart,n=this._cachedDataOpts,o=`animation-${e}`,a=n[o];if(a)return a;let r;if(!1!==s.options.animation){const s=this.chart.config,n=s.datasetAnimationScopeKeys(this._type,e),o=s.getOptionScopes(this.getDataset(),n);r=s.createResolver(o,this.getContext(t,i,e))}const l=new Os(s,r&&r.animations);return r&&r._cacheable&&(n[o]=Object.freeze(l)),l}getSharedOptions(t){if(t.$shared)return this._sharedOptions||(this._sharedOptions=Object.assign({},t))}includeOptions(t,e){return!e||Bs(t)||this.chart._animationsDisabled}_getSharedOptions(t,e){const i=this.resolveDataElementOptions(t,e),s=this._sharedOptions,n=this.getSharedOptions(i),o=this.includeOptions(e,n)||n!==s;return this.updateSharedOptions(n,e,i),{sharedOptions:n,includeOptions:o}}updateElement(t,e,i,s){Bs(s)?Object.assign(t,i):this._resolveAnimations(e,s).update(t,i)}updateSharedOptions(t,e,i){t&&!Bs(e)&&this._resolveAnimations(void 0,e).update(t,i)}_setStyle(t,e,i,s){t.active=s;const n=this.getStyle(e,s);this._resolveAnimations(e,i,s).update(t,{options:!s&&this.getSharedOptions(n)||n})}removeHoverStyle(t,e,i){this._setStyle(t,i,"active",!1)}setHoverStyle(t,e,i){this._setStyle(t,i,"active",!0)}_removeDatasetHoverStyle(){const t=this._cachedMeta.dataset;t&&this._setStyle(t,void 0,"active",!1)}_setDatasetHoverStyle(){const t=this._cachedMeta.dataset;t&&this._setStyle(t,void 0,"active",!0)}_resyncElements(t){const e=this._data,i=this._cachedMeta.data;for(const[t,e,i]of this._syncList)this[t](e,i);this._syncList=[];const s=i.length,n=e.length,o=Math.min(n,s);o&&this.parse(0,o),n>s?this._insertElements(s,n-s,t):n{for(t.length+=e,a=t.length-1;a>=o;a--)t[a]=t[a-e]};for(r(n),a=t;a{s[t]=i[t]&&i[t].active()?i[t]._to:this[t]})),s}}function js(t,e){const i=t.options.ticks,n=function(t){const e=t.options.offset,i=t._tickSize(),s=t._length/i+(e?0:1),n=t._maxLength/i;return Math.floor(Math.min(s,n))}(t),o=Math.min(i.maxTicksLimit||n,n),a=i.major.enabled?function(t){const e=[];let i,s;for(i=0,s=t.length;io)return function(t,e,i,s){let n,o=0,a=i[0];for(s=Math.ceil(s),n=0;nn)return e}return Math.max(n,1)}(a,e,o);if(r>0){let t,i;const n=r>1?Math.round((h-l)/(r-1)):null;for($s(e,c,d,s(n)?0:l-n,l),t=0,i=r-1;t"top"===e||"left"===e?t[e]+i:t[e]-i,Us=(t,e)=>Math.min(e||t,t);function Xs(t,e){const i=[],s=t.length/e,n=t.length;let o=0;for(;oa+r)))return h}function Ks(t){return t.drawTicks?t.tickLength:0}function Gs(t,e){if(!t.display)return 0;const i=Si(t.font,e),s=ki(t.padding);return(n(t.text)?t.text.length:1)*i.lineHeight+s.height}function Zs(t,e,i){let s=ut(t);return(i&&"right"!==e||!i&&"right"===e)&&(s=(t=>"left"===t?"right":"right"===t?"left":t)(s)),s}class Js extends Hs{constructor(t){super(),this.id=t.id,this.type=t.type,this.options=void 0,this.ctx=t.ctx,this.chart=t.chart,this.top=void 0,this.bottom=void 0,this.left=void 0,this.right=void 0,this.width=void 0,this.height=void 0,this._margins={left:0,right:0,top:0,bottom:0},this.maxWidth=void 0,this.maxHeight=void 0,this.paddingTop=void 0,this.paddingBottom=void 0,this.paddingLeft=void 0,this.paddingRight=void 0,this.axis=void 0,this.labelRotation=void 0,this.min=void 0,this.max=void 0,this._range=void 0,this.ticks=[],this._gridLineItems=null,this._labelItems=null,this._labelSizes=null,this._length=0,this._maxLength=0,this._longestTextCache={},this._startPixel=void 0,this._endPixel=void 0,this._reversePixels=!1,this._userMax=void 0,this._userMin=void 0,this._suggestedMax=void 0,this._suggestedMin=void 0,this._ticksLength=0,this._borderValue=0,this._cache={},this._dataLimitsCached=!1,this.$context=void 0}init(t){this.options=t.setContext(this.getContext()),this.axis=t.axis,this._userMin=this.parse(t.min),this._userMax=this.parse(t.max),this._suggestedMin=this.parse(t.suggestedMin),this._suggestedMax=this.parse(t.suggestedMax)}parse(t,e){return t}getUserBounds(){let{_userMin:t,_userMax:e,_suggestedMin:i,_suggestedMax:s}=this;return t=r(t,Number.POSITIVE_INFINITY),e=r(e,Number.NEGATIVE_INFINITY),i=r(i,Number.POSITIVE_INFINITY),s=r(s,Number.NEGATIVE_INFINITY),{min:r(t,i),max:r(e,s),minDefined:a(t),maxDefined:a(e)}}getMinMax(t){let e,{min:i,max:s,minDefined:n,maxDefined:o}=this.getUserBounds();if(n&&o)return{min:i,max:s};const a=this.getMatchingVisibleMetas();for(let r=0,l=a.length;rs?s:i,s=n&&i>s?i:s,{min:r(i,r(s,i)),max:r(s,r(i,s))}}getPadding(){return{left:this.paddingLeft||0,top:this.paddingTop||0,right:this.paddingRight||0,bottom:this.paddingBottom||0}}getTicks(){return this.ticks}getLabels(){const t=this.chart.data;return this.options.labels||(this.isHorizontal()?t.xLabels:t.yLabels)||t.labels||[]}getLabelItems(t=this.chart.chartArea){return this._labelItems||(this._labelItems=this._computeLabelItems(t))}beforeLayout(){this._cache={},this._dataLimitsCached=!1}beforeUpdate(){d(this.options.beforeUpdate,[this])}update(t,e,i){const{beginAtZero:s,grace:n,ticks:o}=this.options,a=o.sampleSize;this.beforeUpdate(),this.maxWidth=t,this.maxHeight=e,this._margins=i=Object.assign({left:0,right:0,top:0,bottom:0},i),this.ticks=null,this._labelSizes=null,this._gridLineItems=null,this._labelItems=null,this.beforeSetDimensions(),this.setDimensions(),this.afterSetDimensions(),this._maxLength=this.isHorizontal()?this.width+i.left+i.right:this.height+i.top+i.bottom,this._dataLimitsCached||(this.beforeDataLimits(),this.determineDataLimits(),this.afterDataLimits(),this._range=Di(this,n,s),this._dataLimitsCached=!0),this.beforeBuildTicks(),this.ticks=this.buildTicks()||[],this.afterBuildTicks();const r=a=n||i<=1||!this.isHorizontal())return void(this.labelRotation=s);const h=this._getLabelSizes(),c=h.widest.width,d=h.highest.height,u=J(this.chart.width-c,0,this.maxWidth);o=t.offset?this.maxWidth/i:u/(i-1),c+6>o&&(o=u/(i-(t.offset?.5:1)),a=this.maxHeight-Ks(t.grid)-e.padding-Gs(t.title,this.chart.options.font),r=Math.sqrt(c*c+d*d),l=Y(Math.min(Math.asin(J((h.highest.height+6)/o,-1,1)),Math.asin(J(a/r,-1,1))-Math.asin(J(d/r,-1,1)))),l=Math.max(s,Math.min(n,l))),this.labelRotation=l}afterCalculateLabelRotation(){d(this.options.afterCalculateLabelRotation,[this])}afterAutoSkip(){}beforeFit(){d(this.options.beforeFit,[this])}fit(){const t={width:0,height:0},{chart:e,options:{ticks:i,title:s,grid:n}}=this,o=this._isVisible(),a=this.isHorizontal();if(o){const o=Gs(s,e.options.font);if(a?(t.width=this.maxWidth,t.height=Ks(n)+o):(t.height=this.maxHeight,t.width=Ks(n)+o),i.display&&this.ticks.length){const{first:e,last:s,widest:n,highest:o}=this._getLabelSizes(),r=2*i.padding,l=$(this.labelRotation),h=Math.cos(l),c=Math.sin(l);if(a){const e=i.mirror?0:c*n.width+h*o.height;t.height=Math.min(this.maxHeight,t.height+e+r)}else{const e=i.mirror?0:h*n.width+c*o.height;t.width=Math.min(this.maxWidth,t.width+e+r)}this._calculatePadding(e,s,c,h)}}this._handleMargins(),a?(this.width=this._length=e.width-this._margins.left-this._margins.right,this.height=t.height):(this.width=t.width,this.height=this._length=e.height-this._margins.top-this._margins.bottom)}_calculatePadding(t,e,i,s){const{ticks:{align:n,padding:o},position:a}=this.options,r=0!==this.labelRotation,l="top"!==a&&"x"===this.axis;if(this.isHorizontal()){const a=this.getPixelForTick(0)-this.left,h=this.right-this.getPixelForTick(this.ticks.length-1);let c=0,d=0;r?l?(c=s*t.width,d=i*e.height):(c=i*t.height,d=s*e.width):"start"===n?d=e.width:"end"===n?c=t.width:"inner"!==n&&(c=t.width/2,d=e.width/2),this.paddingLeft=Math.max((c-a+o)*this.width/(this.width-a),0),this.paddingRight=Math.max((d-h+o)*this.width/(this.width-h),0)}else{let i=e.height/2,s=t.height/2;"start"===n?(i=0,s=t.height):"end"===n&&(i=e.height,s=0),this.paddingTop=i+o,this.paddingBottom=s+o}}_handleMargins(){this._margins&&(this._margins.left=Math.max(this.paddingLeft,this._margins.left),this._margins.top=Math.max(this.paddingTop,this._margins.top),this._margins.right=Math.max(this.paddingRight,this._margins.right),this._margins.bottom=Math.max(this.paddingBottom,this._margins.bottom))}afterFit(){d(this.options.afterFit,[this])}isHorizontal(){const{axis:t,position:e}=this.options;return"top"===e||"bottom"===e||"x"===t}isFullSize(){return this.options.fullSize}_convertTicksToLabels(t){let e,i;for(this.beforeTickToLabelConversion(),this.generateTickLabels(t),e=0,i=t.length;e{const i=t.gc,s=i.length/2;let n;if(s>e){for(n=0;n({width:r[t]||0,height:l[t]||0});return{first:P(0),last:P(e-1),widest:P(k),highest:P(S),widths:r,heights:l}}getLabelForValue(t){return t}getPixelForValue(t,e){return NaN}getValueForPixel(t){}getPixelForTick(t){const e=this.ticks;return t<0||t>e.length-1?null:this.getPixelForValue(e[t].value)}getPixelForDecimal(t){this._reversePixels&&(t=1-t);const e=this._startPixel+t*this._length;return Q(this._alignToPixels?Ae(this.chart,e,0):e)}getDecimalForPixel(t){const e=(t-this._startPixel)/this._length;return this._reversePixels?1-e:e}getBasePixel(){return this.getPixelForValue(this.getBaseValue())}getBaseValue(){const{min:t,max:e}=this;return t<0&&e<0?e:t>0&&e>0?t:0}getContext(t){const e=this.ticks||[];if(t>=0&&ta*s?a/i:r/s:r*s0}_computeGridLineItems(t){const e=this.axis,i=this.chart,s=this.options,{grid:n,position:a,border:r}=s,h=n.offset,c=this.isHorizontal(),d=this.ticks.length+(h?1:0),u=Ks(n),f=[],g=r.setContext(this.getContext()),p=g.display?g.width:0,m=p/2,b=function(t){return Ae(i,t,p)};let x,_,y,v,M,w,k,S,P,D,C,O;if("top"===a)x=b(this.bottom),w=this.bottom-u,S=x-m,D=b(t.top)+m,O=t.bottom;else if("bottom"===a)x=b(this.top),D=t.top,O=b(t.bottom)-m,w=x+m,S=this.top+u;else if("left"===a)x=b(this.right),M=this.right-u,k=x-m,P=b(t.left)+m,C=t.right;else if("right"===a)x=b(this.left),P=t.left,C=b(t.right)-m,M=x+m,k=this.left+u;else if("x"===e){if("center"===a)x=b((t.top+t.bottom)/2+.5);else if(o(a)){const t=Object.keys(a)[0],e=a[t];x=b(this.chart.scales[t].getPixelForValue(e))}D=t.top,O=t.bottom,w=x+m,S=w+u}else if("y"===e){if("center"===a)x=b((t.left+t.right)/2);else if(o(a)){const t=Object.keys(a)[0],e=a[t];x=b(this.chart.scales[t].getPixelForValue(e))}M=x-m,k=M-u,P=t.left,C=t.right}const A=l(s.ticks.maxTicksLimit,d),T=Math.max(1,Math.ceil(d/A));for(_=0;_e.value===t));if(i>=0){return e.setContext(this.getContext(i)).lineWidth}return 0}drawGrid(t){const e=this.options.grid,i=this.ctx,s=this._gridLineItems||(this._gridLineItems=this._computeGridLineItems(t));let n,o;const a=(t,e,s)=>{s.width&&s.color&&(i.save(),i.lineWidth=s.width,i.strokeStyle=s.color,i.setLineDash(s.borderDash||[]),i.lineDashOffset=s.borderDashOffset,i.beginPath(),i.moveTo(t.x,t.y),i.lineTo(e.x,e.y),i.stroke(),i.restore())};if(e.display)for(n=0,o=s.length;n{this.drawBackground(),this.drawGrid(t),this.drawTitle()}},{z:s,draw:()=>{this.drawBorder()}},{z:e,draw:t=>{this.drawLabels(t)}}]:[{z:e,draw:t=>{this.draw(t)}}]}getMatchingVisibleMetas(t){const e=this.chart.getSortedVisibleDatasetMetas(),i=this.axis+"AxisID",s=[];let n,o;for(n=0,o=e.length;n{const s=i.split("."),n=s.pop(),o=[t].concat(s).join("."),a=e[i].split("."),r=a.pop(),l=a.join(".");ue.route(o,n,l,r)}))}(e,t.defaultRoutes);t.descriptors&&ue.describe(e,t.descriptors)}(t,o,i),this.override&&ue.override(t.id,t.overrides)),o}get(t){return this.items[t]}unregister(t){const e=this.items,i=t.id,s=this.scope;i in e&&delete e[i],s&&i in ue[s]&&(delete ue[s][i],this.override&&delete re[i])}}class tn{constructor(){this.controllers=new Qs(Ws,"datasets",!0),this.elements=new Qs(Hs,"elements"),this.plugins=new Qs(Object,"plugins"),this.scales=new Qs(Js,"scales"),this._typedRegistries=[this.controllers,this.scales,this.elements]}add(...t){this._each("register",t)}remove(...t){this._each("unregister",t)}addControllers(...t){this._each("register",t,this.controllers)}addElements(...t){this._each("register",t,this.elements)}addPlugins(...t){this._each("register",t,this.plugins)}addScales(...t){this._each("register",t,this.scales)}getController(t){return this._get(t,this.controllers,"controller")}getElement(t){return this._get(t,this.elements,"element")}getPlugin(t){return this._get(t,this.plugins,"plugin")}getScale(t){return this._get(t,this.scales,"scale")}removeControllers(...t){this._each("unregister",t,this.controllers)}removeElements(...t){this._each("unregister",t,this.elements)}removePlugins(...t){this._each("unregister",t,this.plugins)}removeScales(...t){this._each("unregister",t,this.scales)}_each(t,e,i){[...e].forEach((e=>{const s=i||this._getRegistryForType(e);i||s.isForType(e)||s===this.plugins&&e.id?this._exec(t,s,e):u(e,(e=>{const s=i||this._getRegistryForType(e);this._exec(t,s,e)}))}))}_exec(t,e,i){const s=w(t);d(i["before"+s],[],i),e[t](i),d(i["after"+s],[],i)}_getRegistryForType(t){for(let e=0;et.filter((t=>!e.some((e=>t.plugin.id===e.plugin.id))));this._notify(s(e,i),t,"stop"),this._notify(s(i,e),t,"start")}}function nn(t,e){return e||!1!==t?!0===t?{}:t:null}function on(t,{plugin:e,local:i},s,n){const o=t.pluginScopeKeys(e),a=t.getOptionScopes(s,o);return i&&e.defaults&&a.push(e.defaults),t.createResolver(a,n,[""],{scriptable:!1,indexable:!1,allKeys:!0})}function an(t,e){const i=ue.datasets[t]||{};return((e.datasets||{})[t]||{}).indexAxis||e.indexAxis||i.indexAxis||"x"}function rn(t){if("x"===t||"y"===t||"r"===t)return t}function ln(t,...e){if(rn(t))return t;for(const s of e){const e=s.axis||("top"===(i=s.position)||"bottom"===i?"x":"left"===i||"right"===i?"y":void 0)||t.length>1&&rn(t[0].toLowerCase());if(e)return e}var i;throw new Error(`Cannot determine type of '${t}' axis. Please provide 'axis' or 'position' option.`)}function hn(t,e,i){if(i[e+"AxisID"]===t)return{axis:e}}function cn(t,e){const i=re[t.type]||{scales:{}},s=e.scales||{},n=an(t.type,e),a=Object.create(null);return Object.keys(s).forEach((e=>{const r=s[e];if(!o(r))return console.error(`Invalid scale configuration for scale: ${e}`);if(r._proxy)return console.warn(`Ignoring resolver passed as options for scale: ${e}`);const l=ln(e,r,function(t,e){if(e.data&&e.data.datasets){const i=e.data.datasets.filter((e=>e.xAxisID===t||e.yAxisID===t));if(i.length)return hn(t,"x",i[0])||hn(t,"y",i[0])}return{}}(e,t),ue.scales[r.type]),h=function(t,e){return t===e?"_index_":"_value_"}(l,n),c=i.scales||{};a[e]=x(Object.create(null),[{axis:l},r,c[l],c[h]])})),t.data.datasets.forEach((i=>{const n=i.type||t.type,o=i.indexAxis||an(n,e),r=(re[n]||{}).scales||{};Object.keys(r).forEach((t=>{const e=function(t,e){let i=t;return"_index_"===t?i=e:"_value_"===t&&(i="x"===e?"y":"x"),i}(t,o),n=i[e+"AxisID"]||e;a[n]=a[n]||Object.create(null),x(a[n],[{axis:e},s[n],r[t]])}))})),Object.keys(a).forEach((t=>{const e=a[t];x(e,[ue.scales[e.type],ue.scale])})),a}function dn(t){const e=t.options||(t.options={});e.plugins=l(e.plugins,{}),e.scales=cn(t,e)}function un(t){return(t=t||{}).datasets=t.datasets||[],t.labels=t.labels||[],t}const fn=new Map,gn=new Set;function pn(t,e){let i=fn.get(t);return i||(i=e(),fn.set(t,i),gn.add(i)),i}const mn=(t,e,i)=>{const s=M(e,i);void 0!==s&&t.add(s)};class bn{constructor(t){this._config=function(t){return(t=t||{}).data=un(t.data),dn(t),t}(t),this._scopeCache=new Map,this._resolverCache=new Map}get platform(){return this._config.platform}get type(){return this._config.type}set type(t){this._config.type=t}get data(){return this._config.data}set data(t){this._config.data=un(t)}get options(){return this._config.options}set options(t){this._config.options=t}get plugins(){return this._config.plugins}update(){const t=this._config;this.clearCache(),dn(t)}clearCache(){this._scopeCache.clear(),this._resolverCache.clear()}datasetScopeKeys(t){return pn(t,(()=>[[`datasets.${t}`,""]]))}datasetAnimationScopeKeys(t,e){return pn(`${t}.transition.${e}`,(()=>[[`datasets.${t}.transitions.${e}`,`transitions.${e}`],[`datasets.${t}`,""]]))}datasetElementScopeKeys(t,e){return pn(`${t}-${e}`,(()=>[[`datasets.${t}.elements.${e}`,`datasets.${t}`,`elements.${e}`,""]]))}pluginScopeKeys(t){const e=t.id;return pn(`${this.type}-plugin-${e}`,(()=>[[`plugins.${e}`,...t.additionalOptionScopes||[]]]))}_cachedScopes(t,e){const i=this._scopeCache;let s=i.get(t);return s&&!e||(s=new Map,i.set(t,s)),s}getOptionScopes(t,e,i){const{options:s,type:n}=this,o=this._cachedScopes(t,i),a=o.get(e);if(a)return a;const r=new Set;e.forEach((e=>{t&&(r.add(t),e.forEach((e=>mn(r,t,e)))),e.forEach((t=>mn(r,s,t))),e.forEach((t=>mn(r,re[n]||{},t))),e.forEach((t=>mn(r,ue,t))),e.forEach((t=>mn(r,le,t)))}));const l=Array.from(r);return 0===l.length&&l.push(Object.create(null)),gn.has(e)&&o.set(e,l),l}chartOptionScopes(){const{options:t,type:e}=this;return[t,re[e]||{},ue.datasets[e]||{},{type:e},ue,le]}resolveNamedOptions(t,e,i,s=[""]){const o={$shared:!0},{resolver:a,subPrefixes:r}=xn(this._resolverCache,t,s);let l=a;if(function(t,e){const{isScriptable:i,isIndexable:s}=Ye(t);for(const o of e){const e=i(o),a=s(o),r=(a||e)&&t[o];if(e&&(S(r)||_n(r))||a&&n(r))return!0}return!1}(a,e)){o.$shared=!1;l=$e(a,i=S(i)?i():i,this.createResolver(t,i,r))}for(const t of e)o[t]=l[t];return o}createResolver(t,e,i=[""],s){const{resolver:n}=xn(this._resolverCache,t,i);return o(e)?$e(n,e,void 0,s):n}}function xn(t,e,i){let s=t.get(e);s||(s=new Map,t.set(e,s));const n=i.join();let o=s.get(n);if(!o){o={resolver:je(e,i),subPrefixes:i.filter((t=>!t.toLowerCase().includes("hover")))},s.set(n,o)}return o}const _n=t=>o(t)&&Object.getOwnPropertyNames(t).reduce(((e,i)=>e||S(t[i])),!1);const yn=["top","bottom","left","right","chartArea"];function vn(t,e){return"top"===t||"bottom"===t||-1===yn.indexOf(t)&&"x"===e}function Mn(t,e){return function(i,s){return i[t]===s[t]?i[e]-s[e]:i[t]-s[t]}}function wn(t){const e=t.chart,i=e.options.animation;e.notifyPlugins("afterRender"),d(i&&i.onComplete,[t],e)}function kn(t){const e=t.chart,i=e.options.animation;d(i&&i.onProgress,[t],e)}function Sn(t){return fe()&&"string"==typeof t?t=document.getElementById(t):t&&t.length&&(t=t[0]),t&&t.canvas&&(t=t.canvas),t}const Pn={},Dn=t=>{const e=Sn(t);return Object.values(Pn).filter((t=>t.canvas===e)).pop()};function Cn(t,e,i){const s=Object.keys(t);for(const n of s){const s=+n;if(s>=e){const o=t[n];delete t[n],(i>0||s>e)&&(t[s+i]=o)}}}class On{static defaults=ue;static instances=Pn;static overrides=re;static registry=en;static version="4.3.0";static getChart=Dn;static register(...t){en.add(...t),An()}static unregister(...t){en.remove(...t),An()}constructor(t,e){const s=this.config=new bn(e),n=Sn(t),o=Dn(n);if(o)throw new Error("Canvas is already in use. Chart with ID '"+o.id+"' must be destroyed before the canvas with ID '"+o.canvas.id+"' can be reused.");const a=s.createResolver(s.chartOptionScopes(),this.getContext());this.platform=new(s.platform||ks(n)),this.platform.updateConfig(s);const r=this.platform.acquireContext(n,a.aspectRatio),l=r&&r.canvas,h=l&&l.height,c=l&&l.width;this.id=i(),this.ctx=r,this.canvas=l,this.width=c,this.height=h,this._options=a,this._aspectRatio=this.aspectRatio,this._layers=[],this._metasets=[],this._stacks=void 0,this.boxes=[],this.currentDevicePixelRatio=void 0,this.chartArea=void 0,this._active=[],this._lastEvent=void 0,this._listeners={},this._responsiveListeners=void 0,this._sortedMetasets=[],this.scales={},this._plugins=new sn,this.$proxies={},this._hiddenIndices={},this.attached=!1,this._animationsDisabled=void 0,this.$context=void 0,this._doResize=dt((t=>this.update(t)),a.resizeDelay||0),this._dataChanges=[],Pn[this.id]=this,r&&l?(xt.listen(this,"complete",wn),xt.listen(this,"progress",kn),this._initialize(),this.attached&&this.update()):console.error("Failed to create chart: can't acquire context from the given item")}get aspectRatio(){const{options:{aspectRatio:t,maintainAspectRatio:e},width:i,height:n,_aspectRatio:o}=this;return s(t)?e&&o?o:n?i/n:null:t}get data(){return this.config.data}set data(t){this.config.data=t}get options(){return this._options}set options(t){this.config.options=t}get registry(){return en}_initialize(){return this.notifyPlugins("beforeInit"),this.options.responsive?this.resize():ke(this,this.options.devicePixelRatio),this.bindEvents(),this.notifyPlugins("afterInit"),this}clear(){return Te(this.canvas,this.ctx),this}stop(){return xt.stop(this),this}resize(t,e){xt.running(this)?this._resizeBeforeDraw={width:t,height:e}:this._resize(t,e)}_resize(t,e){const i=this.options,s=this.canvas,n=i.maintainAspectRatio&&this.aspectRatio,o=this.platform.getMaximumSize(s,t,e,n),a=i.devicePixelRatio||this.platform.getDevicePixelRatio(),r=this.width?"resize":"attach";this.width=o.width,this.height=o.height,this._aspectRatio=this.aspectRatio,ke(this,a,!0)&&(this.notifyPlugins("resize",{size:o}),d(i.onResize,[this,o],this),this.attached&&this._doResize(r)&&this.render())}ensureScalesHaveIDs(){u(this.options.scales||{},((t,e)=>{t.id=e}))}buildOrUpdateScales(){const t=this.options,e=t.scales,i=this.scales,s=Object.keys(i).reduce(((t,e)=>(t[e]=!1,t)),{});let n=[];e&&(n=n.concat(Object.keys(e).map((t=>{const i=e[t],s=ln(t,i),n="r"===s,o="x"===s;return{options:i,dposition:n?"chartArea":o?"bottom":"left",dtype:n?"radialLinear":o?"category":"linear"}})))),u(n,(e=>{const n=e.options,o=n.id,a=ln(o,n),r=l(n.type,e.dtype);void 0!==n.position&&vn(n.position,a)===vn(e.dposition)||(n.position=e.dposition),s[o]=!0;let h=null;if(o in i&&i[o].type===r)h=i[o];else{h=new(en.getScale(r))({id:o,type:r,ctx:this.ctx,chart:this}),i[h.id]=h}h.init(n,t)})),u(s,((t,e)=>{t||delete i[e]})),u(i,(t=>{as.configure(this,t,t.options),as.addBox(this,t)}))}_updateMetasets(){const t=this._metasets,e=this.data.datasets.length,i=t.length;if(t.sort(((t,e)=>t.index-e.index)),i>e){for(let t=e;te.length&&delete this._stacks,t.forEach(((t,i)=>{0===e.filter((e=>e===t._dataset)).length&&this._destroyDatasetMeta(i)}))}buildOrUpdateControllers(){const t=[],e=this.data.datasets;let i,s;for(this._removeUnreferencedMetasets(),i=0,s=e.length;i{this.getDatasetMeta(e).controller.reset()}),this)}reset(){this._resetElements(),this.notifyPlugins("reset")}update(t){const e=this.config;e.update();const i=this._options=e.createResolver(e.chartOptionScopes(),this.getContext()),s=this._animationsDisabled=!i.animation;if(this._updateScales(),this._checkEventBindings(),this._updateHiddenIndices(),this._plugins.invalidate(),!1===this.notifyPlugins("beforeUpdate",{mode:t,cancelable:!0}))return;const n=this.buildOrUpdateControllers();this.notifyPlugins("beforeElementsUpdate");let o=0;for(let t=0,e=this.data.datasets.length;t{t.reset()})),this._updateDatasets(t),this.notifyPlugins("afterUpdate",{mode:t}),this._layers.sort(Mn("z","_idx"));const{_active:a,_lastEvent:r}=this;r?this._eventHandler(r,!0):a.length&&this._updateHoverStyles(a,a,!0),this.render()}_updateScales(){u(this.scales,(t=>{as.removeBox(this,t)})),this.ensureScalesHaveIDs(),this.buildOrUpdateScales()}_checkEventBindings(){const t=this.options,e=new Set(Object.keys(this._listeners)),i=new Set(t.events);P(e,i)&&!!this._responsiveListeners===t.responsive||(this.unbindEvents(),this.bindEvents())}_updateHiddenIndices(){const{_hiddenIndices:t}=this,e=this._getUniformDataChanges()||[];for(const{method:i,start:s,count:n}of e){Cn(t,s,"_removeElements"===i?-n:n)}}_getUniformDataChanges(){const t=this._dataChanges;if(!t||!t.length)return;this._dataChanges=[];const e=this.data.datasets.length,i=e=>new Set(t.filter((t=>t[0]===e)).map(((t,e)=>e+","+t.splice(1).join(",")))),s=i(0);for(let t=1;tt.split(","))).map((t=>({method:t[1],start:+t[2],count:+t[3]})))}_updateLayout(t){if(!1===this.notifyPlugins("beforeLayout",{cancelable:!0}))return;as.update(this,this.width,this.height,t);const e=this.chartArea,i=e.width<=0||e.height<=0;this._layers=[],u(this.boxes,(t=>{i&&"chartArea"===t.position||(t.configure&&t.configure(),this._layers.push(...t._layers()))}),this),this._layers.forEach(((t,e)=>{t._idx=e})),this.notifyPlugins("afterLayout")}_updateDatasets(t){if(!1!==this.notifyPlugins("beforeDatasetsUpdate",{mode:t,cancelable:!0})){for(let t=0,e=this.data.datasets.length;t=0;--e)this._drawDataset(t[e]);this.notifyPlugins("afterDatasetsDraw")}_drawDataset(t){const e=this.ctx,i=t._clip,s=!i.disabled,n=function(t){const{xScale:e,yScale:i}=t;if(e&&i)return{left:e.left,right:e.right,top:i.top,bottom:i.bottom}}(t)||this.chartArea,o={meta:t,index:t.index,cancelable:!0};!1!==this.notifyPlugins("beforeDatasetDraw",o)&&(s&&Ie(e,{left:!1===i.left?0:n.left-i.left,right:!1===i.right?this.width:n.right+i.right,top:!1===i.top?0:n.top-i.top,bottom:!1===i.bottom?this.height:n.bottom+i.bottom}),t.controller.draw(),s&&ze(e),o.cancelable=!1,this.notifyPlugins("afterDatasetDraw",o))}isPointInArea(t){return Re(t,this.chartArea,this._minPadding)}getElementsAtEventForMode(t,e,i,s){const n=Xi.modes[e];return"function"==typeof n?n(this,t,i,s):[]}getDatasetMeta(t){const e=this.data.datasets[t],i=this._metasets;let s=i.filter((t=>t&&t._dataset===e)).pop();return s||(s={type:null,data:[],dataset:null,controller:null,hidden:null,xAxisID:null,yAxisID:null,order:e&&e.order||0,index:t,_dataset:e,_parsed:[],_sorted:!1},i.push(s)),s}getContext(){return this.$context||(this.$context=Ci(null,{chart:this,type:"chart"}))}getVisibleDatasetCount(){return this.getSortedVisibleDatasetMetas().length}isDatasetVisible(t){const e=this.data.datasets[t];if(!e)return!1;const i=this.getDatasetMeta(t);return"boolean"==typeof i.hidden?!i.hidden:!e.hidden}setDatasetVisibility(t,e){this.getDatasetMeta(t).hidden=!e}toggleDataVisibility(t){this._hiddenIndices[t]=!this._hiddenIndices[t]}getDataVisibility(t){return!this._hiddenIndices[t]}_updateVisibility(t,e,i){const s=i?"show":"hide",n=this.getDatasetMeta(t),o=n.controller._resolveAnimations(void 0,s);k(e)?(n.data[e].hidden=!i,this.update()):(this.setDatasetVisibility(t,i),o.update(n,{visible:i}),this.update((e=>e.datasetIndex===t?s:void 0)))}hide(t,e){this._updateVisibility(t,e,!1)}show(t,e){this._updateVisibility(t,e,!0)}_destroyDatasetMeta(t){const e=this._metasets[t];e&&e.controller&&e.controller._destroy(),delete this._metasets[t]}_stop(){let t,e;for(this.stop(),xt.remove(this),t=0,e=this.data.datasets.length;t{e.addEventListener(this,i,s),t[i]=s},s=(t,e,i)=>{t.offsetX=e,t.offsetY=i,this._eventHandler(t)};u(this.options.events,(t=>i(t,s)))}bindResponsiveEvents(){this._responsiveListeners||(this._responsiveListeners={});const t=this._responsiveListeners,e=this.platform,i=(i,s)=>{e.addEventListener(this,i,s),t[i]=s},s=(i,s)=>{t[i]&&(e.removeEventListener(this,i,s),delete t[i])},n=(t,e)=>{this.canvas&&this.resize(t,e)};let o;const a=()=>{s("attach",a),this.attached=!0,this.resize(),i("resize",n),i("detach",o)};o=()=>{this.attached=!1,s("resize",n),this._stop(),this._resize(0,0),i("attach",a)},e.isAttached(this.canvas)?a():o()}unbindEvents(){u(this._listeners,((t,e)=>{this.platform.removeEventListener(this,e,t)})),this._listeners={},u(this._responsiveListeners,((t,e)=>{this.platform.removeEventListener(this,e,t)})),this._responsiveListeners=void 0}updateHoverStyle(t,e,i){const s=i?"set":"remove";let n,o,a,r;for("dataset"===e&&(n=this.getDatasetMeta(t[0].datasetIndex),n.controller["_"+s+"DatasetHoverStyle"]()),a=0,r=t.length;a{const i=this.getDatasetMeta(t);if(!i)throw new Error("No dataset found at index "+t);return{datasetIndex:t,element:i.data[e],index:e}}));!f(i,e)&&(this._active=i,this._lastEvent=null,this._updateHoverStyles(i,e))}notifyPlugins(t,e,i){return this._plugins.notify(this,t,e,i)}isPluginEnabled(t){return 1===this._plugins._cache.filter((e=>e.plugin.id===t)).length}_updateHoverStyles(t,e,i){const s=this.options.hover,n=(t,e)=>t.filter((t=>!e.some((e=>t.datasetIndex===e.datasetIndex&&t.index===e.index)))),o=n(e,t),a=i?t:n(t,e);o.length&&this.updateHoverStyle(o,s.mode,!1),a.length&&s.mode&&this.updateHoverStyle(a,s.mode,!0)}_eventHandler(t,e){const i={event:t,replay:e,cancelable:!0,inChartArea:this.isPointInArea(t)},s=e=>(e.options.events||this.options.events).includes(t.native.type);if(!1===this.notifyPlugins("beforeEvent",i,s))return;const n=this._handleEvent(t,e,i.inChartArea);return i.cancelable=!1,this.notifyPlugins("afterEvent",i,s),(n||i.changed)&&this.render(),this}_handleEvent(t,e,i){const{_active:s=[],options:n}=this,o=e,a=this._getActiveElements(t,s,i,o),r=D(t),l=function(t,e,i,s){return i&&"mouseout"!==t.type?s?e:t:null}(t,this._lastEvent,i,r);i&&(this._lastEvent=null,d(n.onHover,[t,a,this],this),r&&d(n.onClick,[t,a,this],this));const h=!f(a,s);return(h||e)&&(this._active=a,this._updateHoverStyles(a,s,e)),this._lastEvent=l,h}_getActiveElements(t,e,i,s){if("mouseout"===t.type)return[];if(!i)return e;const n=this.options.hover;return this.getElementsAtEventForMode(t,n.mode,n,s)}}function An(){return u(On.instances,(t=>t._plugins.invalidate()))}function Tn(){throw new Error("This method is not implemented: Check that a complete date adapter is provided.")}class Ln{static override(t){Object.assign(Ln.prototype,t)}options;constructor(t){this.options=t||{}}init(){}formats(){return Tn()}parse(){return Tn()}format(){return Tn()}add(){return Tn()}diff(){return Tn()}startOf(){return Tn()}endOf(){return Tn()}}var En={_date:Ln};function Rn(t){const e=t.iScale,i=function(t,e){if(!t._cache.$bar){const i=t.getMatchingVisibleMetas(e);let s=[];for(let e=0,n=i.length;et-e)))}return t._cache.$bar}(e,t.type);let s,n,o,a,r=e._length;const l=()=>{32767!==o&&-32768!==o&&(k(a)&&(r=Math.min(r,Math.abs(o-a)||r)),a=o)};for(s=0,n=i.length;sMath.abs(r)&&(l=r,h=a),e[i.axis]=h,e._custom={barStart:l,barEnd:h,start:n,end:o,min:a,max:r}}(t,e,i,s):e[i.axis]=i.parse(t,s),e}function zn(t,e,i,s){const n=t.iScale,o=t.vScale,a=n.getLabels(),r=n===o,l=[];let h,c,d,u;for(h=i,c=i+s;ht.x,i="left",s="right"):(e=t.base"spacing"!==t,_indexable:t=>"spacing"!==t&&!t.startsWith("borderDash")&&!t.startsWith("hoverBorderDash")};static overrides={aspectRatio:1,plugins:{legend:{labels:{generateLabels(t){const e=t.data;if(e.labels.length&&e.datasets.length){const{labels:{pointStyle:i,color:s}}=t.legend.options;return e.labels.map(((e,n)=>{const o=t.getDatasetMeta(0).controller.getStyle(n);return{text:e,fillStyle:o.backgroundColor,strokeStyle:o.borderColor,fontColor:s,lineWidth:o.borderWidth,pointStyle:i,hidden:!t.getDataVisibility(n),index:n}}))}return[]}},onClick(t,e,i){i.chart.toggleDataVisibility(e.index),i.chart.update()}}}};constructor(t,e){super(t,e),this.enableOptionSharing=!0,this.innerRadius=void 0,this.outerRadius=void 0,this.offsetX=void 0,this.offsetY=void 0}linkScales(){}parse(t,e){const i=this.getDataset().data,s=this._cachedMeta;if(!1===this._parsing)s._parsed=i;else{let n,a,r=t=>+i[t];if(o(i[t])){const{key:t="value"}=this._parsing;r=e=>+M(i[e],t)}for(n=t,a=t+e;nZ(t,r,l,!0)?1:Math.max(e,e*i,s,s*i),g=(t,e,s)=>Z(t,r,l,!0)?-1:Math.min(e,e*i,s,s*i),p=f(0,h,d),m=f(E,c,u),b=g(C,h,d),x=g(C+E,c,u);s=(p-b)/2,n=(m-x)/2,o=-(p+b)/2,a=-(m+x)/2}return{ratioX:s,ratioY:n,offsetX:o,offsetY:a}}(u,d,r),b=(i.width-o)/f,x=(i.height-o)/g,_=Math.max(Math.min(b,x)/2,0),y=c(this.options.radius,_),v=(y-Math.max(y*r,0))/this._getVisibleDatasetWeightTotal();this.offsetX=p*y,this.offsetY=m*y,s.total=this.calculateTotal(),this.outerRadius=y-v*this._getRingWeightOffset(this.index),this.innerRadius=Math.max(this.outerRadius-v*l,0),this.updateElements(n,0,n.length,t)}_circumference(t,e){const i=this.options,s=this._cachedMeta,n=this._getCircumference();return e&&i.animation.animateRotate||!this.chart.getDataVisibility(t)||null===s._parsed[t]||s.data[t].hidden?0:this.calculateCircumference(s._parsed[t]*n/O)}updateElements(t,e,i,s){const n="reset"===s,o=this.chart,a=o.chartArea,r=o.options.animation,l=(a.left+a.right)/2,h=(a.top+a.bottom)/2,c=n&&r.animateScale,d=c?0:this.innerRadius,u=c?0:this.outerRadius,{sharedOptions:f,includeOptions:g}=this._getSharedOptions(e,s);let p,m=this._getRotation();for(p=0;p0&&!isNaN(t)?O*(Math.abs(t)/e):0}getLabelAndValue(t){const e=this._cachedMeta,i=this.chart,s=i.data.labels||[],n=ne(e._parsed[t],i.options.locale);return{label:s[t]||"",value:n}}getMaxBorderWidth(t){let e=0;const i=this.chart;let s,n,o,a,r;if(!t)for(s=0,n=i.data.datasets.length;s{const o=t.getDatasetMeta(0).controller.getStyle(n);return{text:e,fillStyle:o.backgroundColor,strokeStyle:o.borderColor,fontColor:s,lineWidth:o.borderWidth,pointStyle:i,hidden:!t.getDataVisibility(n),index:n}}))}return[]}},onClick(t,e,i){i.chart.toggleDataVisibility(e.index),i.chart.update()}}},scales:{r:{type:"radialLinear",angleLines:{display:!1},beginAtZero:!0,grid:{circular:!0},pointLabels:{display:!1},startAngle:0}}};constructor(t,e){super(t,e),this.innerRadius=void 0,this.outerRadius=void 0}getLabelAndValue(t){const e=this._cachedMeta,i=this.chart,s=i.data.labels||[],n=ne(e._parsed[t].r,i.options.locale);return{label:s[t]||"",value:n}}parseObjectData(t,e,i,s){return ii.bind(this)(t,e,i,s)}update(t){const e=this._cachedMeta.data;this._updateRadius(),this.updateElements(e,0,e.length,t)}getMinMax(){const t=this._cachedMeta,e={min:Number.POSITIVE_INFINITY,max:Number.NEGATIVE_INFINITY};return t.data.forEach(((t,i)=>{const s=this.getParsed(i).r;!isNaN(s)&&this.chart.getDataVisibility(i)&&(se.max&&(e.max=s))})),e}_updateRadius(){const t=this.chart,e=t.chartArea,i=t.options,s=Math.min(e.right-e.left,e.bottom-e.top),n=Math.max(s/2,0),o=(n-Math.max(i.cutoutPercentage?n/100*i.cutoutPercentage:1,0))/t.getVisibleDatasetCount();this.outerRadius=n-o*this.index,this.innerRadius=this.outerRadius-o}updateElements(t,e,i,s){const n="reset"===s,o=this.chart,a=o.options.animation,r=this._cachedMeta.rScale,l=r.xCenter,h=r.yCenter,c=r.getIndexAngle(0)-.5*C;let d,u=c;const f=360/this.countVisibleElements();for(d=0;d{!isNaN(this.getParsed(i).r)&&this.chart.getDataVisibility(i)&&e++})),e}_computeAngle(t,e,i){return this.chart.getDataVisibility(t)?$(this.resolveDataElementOptions(t,e).angle||i):0}}var $n=Object.freeze({__proto__:null,BarController:class extends Ws{static id="bar";static defaults={datasetElementType:!1,dataElementType:"bar",categoryPercentage:.8,barPercentage:.9,grouped:!0,animations:{numbers:{type:"number",properties:["x","y","base","width","height"]}}};static overrides={scales:{_index_:{type:"category",offset:!0,grid:{offset:!0}},_value_:{type:"linear",beginAtZero:!0}}};parsePrimitiveData(t,e,i,s){return zn(t,e,i,s)}parseArrayData(t,e,i,s){return zn(t,e,i,s)}parseObjectData(t,e,i,s){const{iScale:n,vScale:o}=t,{xAxisKey:a="x",yAxisKey:r="y"}=this._parsing,l="x"===n.axis?a:r,h="x"===o.axis?a:r,c=[];let d,u,f,g;for(d=i,u=i+s;dt.controller.options.grouped)),o=i.options.stacked,a=[],r=t=>{const i=t.controller.getParsed(e),n=i&&i[t.vScale.axis];if(s(n)||isNaN(n))return!0};for(const i of n)if((void 0===e||!r(i))&&((!1===o||-1===a.indexOf(i.stack)||void 0===o&&void 0===i.stack)&&a.push(i.stack),i.index===t))break;return a.length||a.push(void 0),a}_getStackCount(t){return this._getStacks(void 0,t).length}_getStackIndex(t,e,i){const s=this._getStacks(t,i),n=void 0!==e?s.indexOf(e):-1;return-1===n?s.length-1:n}_getRuler(){const t=this.options,e=this._cachedMeta,i=e.iScale,s=[];let n,o;for(n=0,o=e.data.length;n=i?1:-1)}(u,e,r)*a,f===r&&(b-=u/2);const t=e.getPixelForDecimal(0),s=e.getPixelForDecimal(1),o=Math.min(t,s),h=Math.max(t,s);b=Math.max(Math.min(b,h),o),d=b+u,i&&!c&&(l._stacks[e.axis]._visualValues[n]=e.getValueForPixel(d)-e.getValueForPixel(b))}if(b===e.getPixelForValue(r)){const t=F(u)*e.getLineWidthForValue(r)/2;b+=t,u-=t}return{size:u,base:b,head:d,center:d+u/2}}_calculateBarIndexPixels(t,e){const i=e.scale,n=this.options,o=n.skipNull,a=l(n.maxBarThickness,1/0);let r,h;if(e.grouped){const i=o?this._getStackCount(t):e.stackCount,l="flex"===n.barThickness?function(t,e,i,s){const n=e.pixels,o=n[t];let a=t>0?n[t-1]:null,r=t=0;--i)e=Math.max(e,t[i].size(this.resolveDataElementOptions(i))/2);return e>0&&e}getLabelAndValue(t){const e=this._cachedMeta,i=this.chart.data.labels||[],{xScale:s,yScale:n}=e,o=this.getParsed(t),a=s.getLabelForValue(o.x),r=n.getLabelForValue(o.y),l=o._custom;return{label:i[t]||"",value:"("+a+", "+r+(l?", "+l:"")+")"}}update(t){const e=this._cachedMeta.data;this.updateElements(e,0,e.length,t)}updateElements(t,e,i,s){const n="reset"===s,{iScale:o,vScale:a}=this._cachedMeta,{sharedOptions:r,includeOptions:l}=this._getSharedOptions(e,s),h=o.axis,c=a.axis;for(let d=e;d0&&this.getParsed(e-1);for(let i=0;i<_;++i){const g=t[i],_=b?g:{};if(i=x){_.skip=!0;continue}const v=this.getParsed(i),M=s(v[f]),w=_[u]=a.getPixelForValue(v[u],i),k=_[f]=o||M?r.getBasePixel():r.getPixelForValue(l?this.applyStack(r,v,l):v[f],i);_.skip=isNaN(w)||isNaN(k)||M,_.stop=i>0&&Math.abs(v[u]-y[u])>m,p&&(_.parsed=v,_.raw=h.data[i]),d&&(_.options=c||this.resolveDataElementOptions(i,g.active?"active":n)),b||this.updateElement(g,i,_,n),y=v}}getMaxOverflow(){const t=this._cachedMeta,e=t.dataset,i=e.options&&e.options.borderWidth||0,s=t.data||[];if(!s.length)return i;const n=s[0].size(this.resolveDataElementOptions(0)),o=s[s.length-1].size(this.resolveDataElementOptions(s.length-1));return Math.max(i,n,o)/2}draw(){const t=this._cachedMeta;t.dataset.updateControlPoints(this.chart.chartArea,t.iScale.axis),super.draw()}},PieController:class extends Hn{static id="pie";static defaults={cutout:0,rotation:0,circumference:360,radius:"100%"}},PolarAreaController:jn,RadarController:class extends Ws{static id="radar";static defaults={datasetElementType:"line",dataElementType:"point",indexAxis:"r",showLine:!0,elements:{line:{fill:"start"}}};static overrides={aspectRatio:1,scales:{r:{type:"radialLinear"}}};getLabelAndValue(t){const e=this._cachedMeta.vScale,i=this.getParsed(t);return{label:e.getLabels()[t],value:""+e.getLabelForValue(i[e.axis])}}parseObjectData(t,e,i,s){return ii.bind(this)(t,e,i,s)}update(t){const e=this._cachedMeta,i=e.dataset,s=e.data||[],n=e.iScale.getLabels();if(i.points=s,"resize"!==t){const e=this.resolveDatasetElementOptions(t);this.options.showLine||(e.borderWidth=0);const o={_loop:!0,_fullLoop:n.length===s.length,options:e};this.updateElement(i,void 0,o,t)}this.updateElements(s,0,s.length,t)}updateElements(t,e,i,s){const n=this._cachedMeta.rScale,o="reset"===s;for(let a=e;a0&&this.getParsed(e-1);for(let c=e;c0&&Math.abs(i[f]-_[f])>b,m&&(p.parsed=i,p.raw=h.data[c]),u&&(p.options=d||this.resolveDataElementOptions(c,e.active?"active":n)),x||this.updateElement(e,c,p,n),_=i}this.updateSharedOptions(d,n,c)}getMaxOverflow(){const t=this._cachedMeta,e=t.data||[];if(!this.options.showLine){let t=0;for(let i=e.length-1;i>=0;--i)t=Math.max(t,e[i].size(this.resolveDataElementOptions(i))/2);return t>0&&t}const i=t.dataset,s=i.options&&i.options.borderWidth||0;if(!e.length)return s;const n=e[0].size(this.resolveDataElementOptions(0)),o=e[e.length-1].size(this.resolveDataElementOptions(e.length-1));return Math.max(s,n,o)/2}}});function Yn(t,e,i,s){const n=vi(t.options.borderRadius,["outerStart","outerEnd","innerStart","innerEnd"]);const o=(i-e)/2,a=Math.min(o,s*e/2),r=t=>{const e=(i-Math.min(o,t))*s/2;return J(t,0,Math.min(o,e))};return{outerStart:r(n.outerStart),outerEnd:r(n.outerEnd),innerStart:J(n.innerStart,0,a),innerEnd:J(n.innerEnd,0,a)}}function Un(t,e,i,s){return{x:i+t*Math.cos(e),y:s+t*Math.sin(e)}}function Xn(t,e,i,s,n,o){const{x:a,y:r,startAngle:l,pixelMargin:h,innerRadius:c}=e,d=Math.max(e.outerRadius+s+i-h,0),u=c>0?c+s+i+h:0;let f=0;const g=n-l;if(s){const t=((c>0?c-s:0)+(d>0?d-s:0))/2;f=(g-(0!==t?g*t/(t+s):g))/2}const p=(g-Math.max(.001,g*d-i/C)/d)/2,m=l+p+f,b=n-p-f,{outerStart:x,outerEnd:_,innerStart:y,innerEnd:v}=Yn(e,u,d,b-m),M=d-x,w=d-_,k=m+x/M,S=b-_/w,P=u+y,D=u+v,O=m+y/P,A=b-v/D;if(t.beginPath(),o){const e=(k+S)/2;if(t.arc(a,r,d,k,e),t.arc(a,r,d,e,S),_>0){const e=Un(w,S,a,r);t.arc(e.x,e.y,_,S,b+E)}const i=Un(D,b,a,r);if(t.lineTo(i.x,i.y),v>0){const e=Un(D,A,a,r);t.arc(e.x,e.y,v,b+E,A+Math.PI)}const s=(b-v/u+(m+y/u))/2;if(t.arc(a,r,u,b-v/u,s,!0),t.arc(a,r,u,s,m+y/u,!0),y>0){const e=Un(P,O,a,r);t.arc(e.x,e.y,y,O+Math.PI,m-E)}const n=Un(M,m,a,r);if(t.lineTo(n.x,n.y),x>0){const e=Un(M,k,a,r);t.arc(e.x,e.y,x,m-E,k)}}else{t.moveTo(a,r);const e=Math.cos(k)*d+a,i=Math.sin(k)*d+r;t.lineTo(e,i);const s=Math.cos(S)*d+a,n=Math.sin(S)*d+r;t.lineTo(s,n)}t.closePath()}function qn(t,e,i,s,n){const{fullCircles:o,startAngle:a,circumference:r,options:l}=e,{borderWidth:h,borderJoinStyle:c,borderDash:d,borderDashOffset:u}=l,f="inner"===l.borderAlign;if(!h)return;t.setLineDash(d||[]),t.lineDashOffset=u,f?(t.lineWidth=2*h,t.lineJoin=c||"round"):(t.lineWidth=h,t.lineJoin=c||"bevel");let g=e.endAngle;if(o){Xn(t,e,i,s,g,n);for(let e=0;en?(h=n/l,t.arc(o,a,l,i+h,s-h,!0)):t.arc(o,a,n,i+E,s-E),t.closePath(),t.clip()}(t,e,g),o||(Xn(t,e,i,s,g,n),t.stroke())}function Kn(t,e,i=e){t.lineCap=l(i.borderCapStyle,e.borderCapStyle),t.setLineDash(l(i.borderDash,e.borderDash)),t.lineDashOffset=l(i.borderDashOffset,e.borderDashOffset),t.lineJoin=l(i.borderJoinStyle,e.borderJoinStyle),t.lineWidth=l(i.borderWidth,e.borderWidth),t.strokeStyle=l(i.borderColor,e.borderColor)}function Gn(t,e,i){t.lineTo(i.x,i.y)}function Zn(t,e,i={}){const s=t.length,{start:n=0,end:o=s-1}=i,{start:a,end:r}=e,l=Math.max(n,a),h=Math.min(o,r),c=nr&&o>r;return{count:s,start:l,loop:e.loop,ilen:h(a+(h?r-t:t))%o,_=()=>{f!==g&&(t.lineTo(m,g),t.lineTo(m,f),t.lineTo(m,p))};for(l&&(d=n[x(0)],t.moveTo(d.x,d.y)),c=0;c<=r;++c){if(d=n[x(c)],d.skip)continue;const e=d.x,i=d.y,s=0|e;s===u?(ig&&(g=i),m=(b*m+e)/++b):(_(),t.lineTo(e,i),u=s,b=0,f=g=i),p=i}_()}function to(t){const e=t.options,i=e.borderDash&&e.borderDash.length;return!(t._decimated||t._loop||e.tension||"monotone"===e.cubicInterpolationMode||e.stepped||i)?Qn:Jn}const eo="function"==typeof Path2D;function io(t,e,i,s){eo&&!e.options.segment?function(t,e,i,s){let n=e._path;n||(n=e._path=new Path2D,e.path(n,i,s)&&n.closePath()),Kn(t,e.options),t.stroke(n)}(t,e,i,s):function(t,e,i,s){const{segments:n,options:o}=e,a=to(e);for(const r of n)Kn(t,o,r.style),t.beginPath(),a(t,e,r,{start:i,end:i+s-1})&&t.closePath(),t.stroke()}(t,e,i,s)}class so extends Hs{static id="line";static defaults={borderCapStyle:"butt",borderDash:[],borderDashOffset:0,borderJoinStyle:"miter",borderWidth:3,capBezierPoints:!0,cubicInterpolationMode:"default",fill:!1,spanGaps:!1,stepped:!1,tension:0};static defaultRoutes={backgroundColor:"backgroundColor",borderColor:"borderColor"};static descriptors={_scriptable:!0,_indexable:t=>"borderDash"!==t&&"fill"!==t};constructor(t){super(),this.animated=!0,this.options=void 0,this._chart=void 0,this._loop=void 0,this._fullLoop=void 0,this._path=void 0,this._points=void 0,this._segments=void 0,this._decimated=!1,this._pointsUpdated=!1,this._datasetIndex=void 0,t&&Object.assign(this,t)}updateControlPoints(t,e){const i=this.options;if((i.tension||"monotone"===i.cubicInterpolationMode)&&!i.stepped&&!this._pointsUpdated){const s=i.spanGaps?this._loop:this._fullLoop;hi(this._points,i,t,s,e),this._pointsUpdated=!0}}set points(t){this._points=t,delete this._segments,delete this._path,this._pointsUpdated=!1}get points(){return this._points}get segments(){return this._segments||(this._segments=zi(this,this.options.segment))}first(){const t=this.segments,e=this.points;return t.length&&e[t[0].start]}last(){const t=this.segments,e=this.points,i=t.length;return i&&e[t[i-1].end]}interpolate(t,e){const i=this.options,s=t[e],n=this.points,o=Ii(this,{property:e,start:s,end:s});if(!o.length)return;const a=[],r=function(t){return t.stepped?pi:t.tension||"monotone"===t.cubicInterpolationMode?mi:gi}(i);let l,h;for(l=0,h=o.length;l"borderDash"!==t};circumference;endAngle;fullCircles;innerRadius;outerRadius;pixelMargin;startAngle;constructor(t){super(),this.options=void 0,this.circumference=void 0,this.startAngle=void 0,this.endAngle=void 0,this.innerRadius=void 0,this.outerRadius=void 0,this.pixelMargin=0,this.fullCircles=0,t&&Object.assign(this,t)}inRange(t,e,i){const s=this.getProps(["x","y"],i),{angle:n,distance:o}=X(s,{x:t,y:e}),{startAngle:a,endAngle:r,innerRadius:h,outerRadius:c,circumference:d}=this.getProps(["startAngle","endAngle","innerRadius","outerRadius","circumference"],i),u=(this.options.spacing+this.options.borderWidth)/2,f=l(d,r-a)>=O||Z(n,a,r),g=tt(o,h+u,c+u);return f&&g}getCenterPoint(t){const{x:e,y:i,startAngle:s,endAngle:n,innerRadius:o,outerRadius:a}=this.getProps(["x","y","startAngle","endAngle","innerRadius","outerRadius"],t),{offset:r,spacing:l}=this.options,h=(s+n)/2,c=(o+a+l+r)/2;return{x:e+Math.cos(h)*c,y:i+Math.sin(h)*c}}tooltipPosition(t){return this.getCenterPoint(t)}draw(t){const{options:e,circumference:i}=this,s=(e.offset||0)/4,n=(e.spacing||0)/2,o=e.circular;if(this.pixelMargin="inner"===e.borderAlign?.33:0,this.fullCircles=i>O?Math.floor(i/O):0,0===i||this.innerRadius<0||this.outerRadius<0)return;t.save();const a=(this.startAngle+this.endAngle)/2;t.translate(Math.cos(a)*s,Math.sin(a)*s);const r=s*(1-Math.sin(Math.min(C,i||0)));t.fillStyle=e.backgroundColor,t.strokeStyle=e.borderColor,function(t,e,i,s,n){const{fullCircles:o,startAngle:a,circumference:r}=e;let l=e.endAngle;if(o){Xn(t,e,i,s,l,n);for(let e=0;e("string"==typeof e?(i=t.push(e)-1,s.unshift({index:i,label:e})):isNaN(e)&&(i=null),i))(t,e,i,s);return n!==t.lastIndexOf(e)?i:n}function go(t){const e=this.getLabels();return t>=0&&ts=e?s:t,a=t=>n=i?n:t;if(t){const t=F(s),e=F(n);t<0&&e<0?a(0):t>0&&e>0&&o(0)}if(s===n){let e=0===n?1:Math.abs(.05*n);a(n+e),t||o(s-e)}this.min=s,this.max=n}getTickLimit(){const t=this.options.ticks;let e,{maxTicksLimit:i,stepSize:s}=t;return s?(e=Math.ceil(this.max/s)-Math.floor(this.min/s)+1,e>1e3&&(console.warn(`scales.${this.id}.ticks.stepSize: ${s} would result generating up to ${e} ticks. Limiting to 1000.`),e=1e3)):(e=this.computeTickLimit(),i=i||11),i&&(e=Math.min(i,e)),e}computeTickLimit(){return Number.POSITIVE_INFINITY}buildTicks(){const t=this.options,e=t.ticks;let i=this.getTickLimit();i=Math.max(2,i);const n=function(t,e){const i=[],{bounds:n,step:o,min:a,max:r,precision:l,count:h,maxTicks:c,maxDigits:d,includeBounds:u}=t,f=o||1,g=c-1,{min:p,max:m}=e,b=!s(a),x=!s(r),_=!s(h),y=(m-p)/(d+1);let v,M,w,k,S=B((m-p)/g/f)*f;if(S<1e-14&&!b&&!x)return[{value:p},{value:m}];k=Math.ceil(m/S)-Math.floor(p/S),k>g&&(S=B(k*S/g/f)*f),s(l)||(v=Math.pow(10,l),S=Math.ceil(S*v)/v),"ticks"===n?(M=Math.floor(p/S)*S,w=Math.ceil(m/S)*S):(M=p,w=m),b&&x&&o&&H((r-a)/o,S/1e3)?(k=Math.round(Math.min((r-a)/S,c)),S=(r-a)/k,M=a,w=r):_?(M=b?a:M,w=x?r:w,k=h-1,S=(w-M)/k):(k=(w-M)/S,k=V(k,Math.round(k),S/1e3)?Math.round(k):Math.ceil(k));const P=Math.max(U(S),U(M));v=Math.pow(10,s(l)?P:l),M=Math.round(M*v)/v,w=Math.round(w*v)/v;let D=0;for(b&&(u&&M!==a?(i.push({value:a}),Mr)break;i.push({value:t})}return x&&u&&w!==r?i.length&&V(i[i.length-1].value,r,po(r,y,t))?i[i.length-1].value=r:i.push({value:r}):x&&w!==r||i.push({value:w}),i}({maxTicks:i,bounds:t.bounds,min:t.min,max:t.max,precision:e.precision,step:e.stepSize,count:e.count,maxDigits:this._maxDigits(),horizontal:this.isHorizontal(),minRotation:e.minRotation||0,includeBounds:!1!==e.includeBounds},this._range||this);return"ticks"===t.bounds&&j(n,this,"value"),t.reverse?(n.reverse(),this.start=this.max,this.end=this.min):(this.start=this.min,this.end=this.max),n}configure(){const t=this.ticks;let e=this.min,i=this.max;if(super.configure(),this.options.offset&&t.length){const s=(i-e)/Math.max(t.length-1,1)/2;e-=s,i+=s}this._startValue=e,this._endValue=i,this._valueRange=i-e}getLabelForValue(t){return ne(t,this.chart.options.locale,this.options.ticks.format)}}class bo extends mo{static id="linear";static defaults={ticks:{callback:ae.formatters.numeric}};determineDataLimits(){const{min:t,max:e}=this.getMinMax(!0);this.min=a(t)?t:0,this.max=a(e)?e:1,this.handleTickRangeOptions()}computeTickLimit(){const t=this.isHorizontal(),e=t?this.width:this.height,i=$(this.options.ticks.minRotation),s=(t?Math.sin(i):Math.cos(i))||.001,n=this._resolveTickFontOptions(0);return Math.ceil(e/Math.min(40,n.lineHeight/s))}getPixelForValue(t){return null===t?NaN:this.getPixelForDecimal((t-this._startValue)/this._valueRange)}getValueForPixel(t){return this._startValue+this.getDecimalForPixel(t)*this._valueRange}}const xo=t=>Math.floor(z(t)),_o=(t,e)=>Math.pow(10,xo(t)+e);function yo(t){return 1===t/Math.pow(10,xo(t))}function vo(t,e,i){const s=Math.pow(10,i),n=Math.floor(t/s);return Math.ceil(e/s)-n}function Mo(t,{min:e,max:i}){e=r(t.min,e);const s=[],n=xo(e);let o=function(t,e){let i=xo(e-t);for(;vo(t,e,i)>10;)i++;for(;vo(t,e,i)<10;)i--;return Math.min(i,xo(t))}(e,i),a=o<0?Math.pow(10,Math.abs(o)):1;const l=Math.pow(10,o),h=n>o?Math.pow(10,n):0,c=Math.round((e-h)*a)/a,d=Math.floor((e-h)/l/10)*l*10;let u=Math.floor((c-d)/Math.pow(10,o)),f=r(t.min,Math.round((h+d+u*Math.pow(10,o))*a)/a);for(;f=10?u=u<15?15:20:u++,u>=20&&(o++,u=2,a=o>=0?1:a),f=Math.round((h+d+u*Math.pow(10,o))*a)/a;const g=r(t.max,f);return s.push({value:g,major:yo(g),significand:u}),s}class wo extends Js{static id="logarithmic";static defaults={ticks:{callback:ae.formatters.logarithmic,major:{enabled:!0}}};constructor(t){super(t),this.start=void 0,this.end=void 0,this._startValue=void 0,this._valueRange=0}parse(t,e){const i=mo.prototype.parse.apply(this,[t,e]);if(0!==i)return a(i)&&i>0?i:null;this._zero=!0}determineDataLimits(){const{min:t,max:e}=this.getMinMax(!0);this.min=a(t)?Math.max(0,t):null,this.max=a(e)?Math.max(0,e):null,this.options.beginAtZero&&(this._zero=!0),this._zero&&this.min!==this._suggestedMin&&!a(this._userMin)&&(this.min=t===_o(this.min,0)?_o(this.min,-1):_o(this.min,0)),this.handleTickRangeOptions()}handleTickRangeOptions(){const{minDefined:t,maxDefined:e}=this.getUserBounds();let i=this.min,s=this.max;const n=e=>i=t?i:e,o=t=>s=e?s:t;i===s&&(i<=0?(n(1),o(10)):(n(_o(i,-1)),o(_o(s,1)))),i<=0&&n(_o(s,-1)),s<=0&&o(_o(i,1)),this.min=i,this.max=s}buildTicks(){const t=this.options,e=Mo({min:this._userMin,max:this._userMax},this);return"ticks"===t.bounds&&j(e,this,"value"),t.reverse?(e.reverse(),this.start=this.max,this.end=this.min):(this.start=this.min,this.end=this.max),e}getLabelForValue(t){return void 0===t?"0":ne(t,this.chart.options.locale,this.options.ticks.format)}configure(){const t=this.min;super.configure(),this._startValue=z(t),this._valueRange=z(this.max)-z(t)}getPixelForValue(t){return void 0!==t&&0!==t||(t=this.min),null===t||isNaN(t)?NaN:this.getPixelForDecimal(t===this.min?0:(z(t)-this._startValue)/this._valueRange)}getValueForPixel(t){const e=this.getDecimalForPixel(t);return Math.pow(10,this._startValue+e*this._valueRange)}}function ko(t){const e=t.ticks;if(e.display&&t.display){const t=ki(e.backdropPadding);return l(e.font&&e.font.size,ue.font.size)+t.height}return 0}function So(t,e,i,s,n){return t===s||t===n?{start:e-i/2,end:e+i/2}:tn?{start:e-i,end:e}:{start:e,end:e+i}}function Po(t){const e={l:t.left+t._padding.left,r:t.right-t._padding.right,t:t.top+t._padding.top,b:t.bottom-t._padding.bottom},i=Object.assign({},e),s=[],o=[],a=t._pointLabels.length,r=t.options.pointLabels,l=r.centerPointLabels?C/a:0;for(let u=0;ue.r&&(r=(s.end-e.r)/o,t.r=Math.max(t.r,e.r+r)),n.starte.b&&(l=(n.end-e.b)/a,t.b=Math.max(t.b,e.b+l))}function Co(t,e,i){const s=t.drawingArea,{extra:n,additionalAngle:o,padding:a,size:r}=i,l=t.getPointPosition(e,s+n+a,o),h=Math.round(Y(G(l.angle+E))),c=function(t,e,i){90===i||270===i?t-=e/2:(i>270||i<90)&&(t-=e);return t}(l.y,r.h,h),d=function(t){if(0===t||180===t)return"center";if(t<180)return"left";return"right"}(h),u=function(t,e,i){"right"===i?t-=e:"center"===i&&(t-=e/2);return t}(l.x,r.w,d);return{visible:!0,x:l.x,y:c,textAlign:d,left:u,top:c,right:u+r.w,bottom:c+r.h}}function Oo(t,e){if(!e)return!0;const{left:i,top:s,right:n,bottom:o}=t;return!(Re({x:i,y:s},e)||Re({x:i,y:o},e)||Re({x:n,y:s},e)||Re({x:n,y:o},e))}function Ao(t,e,i){const{left:n,top:o,right:a,bottom:r}=i,{backdropColor:l}=e;if(!s(l)){const i=wi(e.borderRadius),s=ki(e.backdropPadding);t.fillStyle=l;const h=n-s.left,c=o-s.top,d=a-n+s.width,u=r-o+s.height;Object.values(i).some((t=>0!==t))?(t.beginPath(),He(t,{x:h,y:c,w:d,h:u,radius:i}),t.fill()):t.fillRect(h,c,d,u)}}function To(t,e,i,s){const{ctx:n}=t;if(i)n.arc(t.xCenter,t.yCenter,e,0,O);else{let i=t.getPointPosition(0,e);n.moveTo(i.x,i.y);for(let o=1;ot,padding:5,centerPointLabels:!1}};static defaultRoutes={"angleLines.color":"borderColor","pointLabels.color":"color","ticks.color":"color"};static descriptors={angleLines:{_fallback:"grid"}};constructor(t){super(t),this.xCenter=void 0,this.yCenter=void 0,this.drawingArea=void 0,this._pointLabels=[],this._pointLabelItems=[]}setDimensions(){const t=this._padding=ki(ko(this.options)/2),e=this.width=this.maxWidth-t.width,i=this.height=this.maxHeight-t.height;this.xCenter=Math.floor(this.left+e/2+t.left),this.yCenter=Math.floor(this.top+i/2+t.top),this.drawingArea=Math.floor(Math.min(e,i)/2)}determineDataLimits(){const{min:t,max:e}=this.getMinMax(!1);this.min=a(t)&&!isNaN(t)?t:0,this.max=a(e)&&!isNaN(e)?e:0,this.handleTickRangeOptions()}computeTickLimit(){return Math.ceil(this.drawingArea/ko(this.options))}generateTickLabels(t){mo.prototype.generateTickLabels.call(this,t),this._pointLabels=this.getLabels().map(((t,e)=>{const i=d(this.options.pointLabels.callback,[t,e],this);return i||0===i?i:""})).filter(((t,e)=>this.chart.getDataVisibility(e)))}fit(){const t=this.options;t.display&&t.pointLabels.display?Po(this):this.setCenterPoint(0,0,0,0)}setCenterPoint(t,e,i,s){this.xCenter+=Math.floor((t-e)/2),this.yCenter+=Math.floor((i-s)/2),this.drawingArea-=Math.min(this.drawingArea/2,Math.max(t,e,i,s))}getIndexAngle(t){return G(t*(O/(this._pointLabels.length||1))+$(this.options.startAngle||0))}getDistanceFromCenterForValue(t){if(s(t))return NaN;const e=this.drawingArea/(this.max-this.min);return this.options.reverse?(this.max-t)*e:(t-this.min)*e}getValueForDistanceFromCenter(t){if(s(t))return NaN;const e=t/(this.drawingArea/(this.max-this.min));return this.options.reverse?this.max-e:this.min+e}getPointLabelContext(t){const e=this._pointLabels||[];if(t>=0&&t=0;n--){const e=t._pointLabelItems[n];if(!e.visible)continue;const o=s.setContext(t.getPointLabelContext(n));Ao(i,o,e);const a=Si(o.font),{x:r,y:l,textAlign:h}=e;We(i,t._pointLabels[n],r,l+a.lineHeight/2,a,{color:o.color,textAlign:h,textBaseline:"middle"})}}(this,o),s.display&&this.ticks.forEach(((t,e)=>{if(0!==e){r=this.getDistanceFromCenterForValue(t.value);const i=this.getContext(e),a=s.setContext(i),l=n.setContext(i);!function(t,e,i,s,n){const o=t.ctx,a=e.circular,{color:r,lineWidth:l}=e;!a&&!s||!r||!l||i<0||(o.save(),o.strokeStyle=r,o.lineWidth=l,o.setLineDash(n.dash),o.lineDashOffset=n.dashOffset,o.beginPath(),To(t,i,a,s),o.closePath(),o.stroke(),o.restore())}(this,a,r,o,l)}})),i.display){for(t.save(),a=o-1;a>=0;a--){const s=i.setContext(this.getPointLabelContext(a)),{color:n,lineWidth:o}=s;o&&n&&(t.lineWidth=o,t.strokeStyle=n,t.setLineDash(s.borderDash),t.lineDashOffset=s.borderDashOffset,r=this.getDistanceFromCenterForValue(e.ticks.reverse?this.min:this.max),l=this.getPointPosition(a,r),t.beginPath(),t.moveTo(this.xCenter,this.yCenter),t.lineTo(l.x,l.y),t.stroke())}t.restore()}}drawBorder(){}drawLabels(){const t=this.ctx,e=this.options,i=e.ticks;if(!i.display)return;const s=this.getIndexAngle(0);let n,o;t.save(),t.translate(this.xCenter,this.yCenter),t.rotate(s),t.textAlign="center",t.textBaseline="middle",this.ticks.forEach(((s,a)=>{if(0===a&&!e.reverse)return;const r=i.setContext(this.getContext(a)),l=Si(r.font);if(n=this.getDistanceFromCenterForValue(this.ticks[a].value),r.showLabelBackdrop){t.font=l.string,o=t.measureText(s.label).width,t.fillStyle=r.backdropColor;const e=ki(r.backdropPadding);t.fillRect(-o/2-e.left,-n-l.size/2-e.top,o+e.width,l.size+e.height)}We(t,s.label,0,-n,l,{color:r.color})})),t.restore()}drawTitle(){}}const Eo={millisecond:{common:!0,size:1,steps:1e3},second:{common:!0,size:1e3,steps:60},minute:{common:!0,size:6e4,steps:60},hour:{common:!0,size:36e5,steps:24},day:{common:!0,size:864e5,steps:30},week:{common:!1,size:6048e5,steps:4},month:{common:!0,size:2628e6,steps:12},quarter:{common:!1,size:7884e6,steps:4},year:{common:!0,size:3154e7}},Ro=Object.keys(Eo);function Io(t,e){return t-e}function zo(t,e){if(s(e))return null;const i=t._adapter,{parser:n,round:o,isoWeekday:r}=t._parseOpts;let l=e;return"function"==typeof n&&(l=n(l)),a(l)||(l="string"==typeof n?i.parse(l,n):i.parse(l)),null===l?null:(o&&(l="week"!==o||!W(r)&&!0!==r?i.startOf(l,o):i.startOf(l,"isoWeek",r)),+l)}function Fo(t,e,i,s){const n=Ro.length;for(let o=Ro.indexOf(t);o=e?i[s]:i[n]]=!0}}else t[e]=!0}function Bo(t,e,i){const s=[],n={},o=e.length;let a,r;for(a=0;a=0&&(e[l].major=!0);return e}(t,s,n,i):s}class No extends Js{static id="time";static defaults={bounds:"data",adapters:{},time:{parser:!1,unit:!1,round:!1,isoWeekday:!1,minUnit:"millisecond",displayFormats:{}},ticks:{source:"auto",callback:!1,major:{enabled:!1}}};constructor(t){super(t),this._cache={data:[],labels:[],all:[]},this._unit="day",this._majorUnit=void 0,this._offsets={},this._normalized=!1,this._parseOpts=void 0}init(t,e={}){const i=t.time||(t.time={}),s=this._adapter=new En._date(t.adapters.date);s.init(e),x(i.displayFormats,s.formats()),this._parseOpts={parser:i.parser,round:i.round,isoWeekday:i.isoWeekday},super.init(t),this._normalized=e.normalized}parse(t,e){return void 0===t?null:zo(this,t)}beforeLayout(){super.beforeLayout(),this._cache={data:[],labels:[],all:[]}}determineDataLimits(){const t=this.options,e=this._adapter,i=t.time.unit||"day";let{min:s,max:n,minDefined:o,maxDefined:r}=this.getUserBounds();function l(t){o||isNaN(t.min)||(s=Math.min(s,t.min)),r||isNaN(t.max)||(n=Math.max(n,t.max))}o&&r||(l(this._getLabelBounds()),"ticks"===t.bounds&&"labels"===t.ticks.source||l(this.getMinMax(!1))),s=a(s)&&!isNaN(s)?s:+e.startOf(Date.now(),i),n=a(n)&&!isNaN(n)?n:+e.endOf(Date.now(),i)+1,this.min=Math.min(s,n-1),this.max=Math.max(s+1,n)}_getLabelBounds(){const t=this.getLabelTimestamps();let e=Number.POSITIVE_INFINITY,i=Number.NEGATIVE_INFINITY;return t.length&&(e=t[0],i=t[t.length-1]),{min:e,max:i}}buildTicks(){const t=this.options,e=t.time,i=t.ticks,s="labels"===i.source?this.getLabelTimestamps():this._generate();"ticks"===t.bounds&&s.length&&(this.min=this._userMin||s[0],this.max=this._userMax||s[s.length-1]);const n=this.min,o=nt(s,n,this.max);return this._unit=e.unit||(i.autoSkip?Fo(e.minUnit,this.min,this.max,this._getLabelCapacity(n)):function(t,e,i,s,n){for(let o=Ro.length-1;o>=Ro.indexOf(i);o--){const i=Ro[o];if(Eo[i].common&&t._adapter.diff(n,s,i)>=e-1)return i}return Ro[i?Ro.indexOf(i):0]}(this,o.length,e.minUnit,this.min,this.max)),this._majorUnit=i.major.enabled&&"year"!==this._unit?function(t){for(let e=Ro.indexOf(t)+1,i=Ro.length;e+t.value)))}initOffsets(t=[]){let e,i,s=0,n=0;this.options.offset&&t.length&&(e=this.getDecimalForValue(t[0]),s=1===t.length?1-e:(this.getDecimalForValue(t[1])-e)/2,i=this.getDecimalForValue(t[t.length-1]),n=1===t.length?i:(i-this.getDecimalForValue(t[t.length-2]))/2);const o=t.length<3?.5:.25;s=J(s,0,o),n=J(n,0,o),this._offsets={start:s,end:n,factor:1/(s+1+n)}}_generate(){const t=this._adapter,e=this.min,i=this.max,s=this.options,n=s.time,o=n.unit||Fo(n.minUnit,e,i,this._getLabelCapacity(e)),a=l(s.ticks.stepSize,1),r="week"===o&&n.isoWeekday,h=W(r)||!0===r,c={};let d,u,f=e;if(h&&(f=+t.startOf(f,"isoWeek",r)),f=+t.startOf(f,h?"day":o),t.diff(i,e,o)>1e5*a)throw new Error(e+" and "+i+" are too far apart with stepSize of "+a+" "+o);const g="data"===s.ticks.source&&this.getDataTimestamps();for(d=f,u=0;dt-e)).map((t=>+t))}getLabelForValue(t){const e=this._adapter,i=this.options.time;return i.tooltipFormat?e.format(t,i.tooltipFormat):e.format(t,i.displayFormats.datetime)}format(t,e){const i=this.options.time.displayFormats,s=this._unit,n=e||i[s];return this._adapter.format(t,n)}_tickFormatFunction(t,e,i,s){const n=this.options,o=n.ticks.callback;if(o)return d(o,[t,e,i],this);const a=n.time.displayFormats,r=this._unit,l=this._majorUnit,h=r&&a[r],c=l&&a[l],u=i[e],f=l&&c&&u&&u.major;return this._adapter.format(t,s||(f?c:h))}generateTickLabels(t){let e,i,s;for(e=0,i=t.length;e0?a:1}getDataTimestamps(){let t,e,i=this._cache.data||[];if(i.length)return i;const s=this.getMatchingVisibleMetas();if(this._normalized&&s.length)return this._cache.data=s[0].controller.getAllParsedValues(this);for(t=0,e=s.length;t=t[r].pos&&e<=t[l].pos&&({lo:r,hi:l}=it(t,"pos",e)),({pos:s,time:o}=t[r]),({pos:n,time:a}=t[l])):(e>=t[r].time&&e<=t[l].time&&({lo:r,hi:l}=it(t,"time",e)),({time:s,pos:o}=t[r]),({time:n,pos:a}=t[l]));const h=n-s;return h?o+(a-o)*(e-s)/h:o}var Ho=Object.freeze({__proto__:null,CategoryScale:class extends Js{static id="category";static defaults={ticks:{callback:go}};constructor(t){super(t),this._startValue=void 0,this._valueRange=0,this._addedLabels=[]}init(t){const e=this._addedLabels;if(e.length){const t=this.getLabels();for(const{index:i,label:s}of e)t[i]===s&&t.splice(i,1);this._addedLabels=[]}super.init(t)}parse(t,e){if(s(t))return null;const i=this.getLabels();return((t,e)=>null===t?null:J(Math.round(t),0,e))(e=isFinite(e)&&i[e]===t?e:fo(i,t,l(e,t),this._addedLabels),i.length-1)}determineDataLimits(){const{minDefined:t,maxDefined:e}=this.getUserBounds();let{min:i,max:s}=this.getMinMax(!0);"ticks"===this.options.bounds&&(t||(i=0),e||(s=this.getLabels().length-1)),this.min=i,this.max=s}buildTicks(){const t=this.min,e=this.max,i=this.options.offset,s=[];let n=this.getLabels();n=0===t&&e===n.length-1?n:n.slice(t,e+1),this._valueRange=Math.max(n.length-(i?0:1),1),this._startValue=this.min-(i?.5:0);for(let i=t;i<=e;i++)s.push({value:i});return s}getLabelForValue(t){return go.call(this,t)}configure(){super.configure(),this.isHorizontal()||(this._reversePixels=!this._reversePixels)}getPixelForValue(t){return"number"!=typeof t&&(t=this.parse(t)),null===t?NaN:this.getPixelForDecimal((t-this._startValue)/this._valueRange)}getPixelForTick(t){const e=this.ticks;return t<0||t>e.length-1?null:this.getPixelForValue(e[t].value)}getValueForPixel(t){return Math.round(this._startValue+this.getDecimalForPixel(t)*this._valueRange)}getBasePixel(){return this.bottom}},LinearScale:bo,LogarithmicScale:wo,RadialLinearScale:Lo,TimeScale:No,TimeSeriesScale:class extends No{static id="timeseries";static defaults=No.defaults;constructor(t){super(t),this._table=[],this._minPos=void 0,this._tableRange=void 0}initOffsets(){const t=this._getTimestampsForTable(),e=this._table=this.buildLookupTable(t);this._minPos=Wo(e,this.min),this._tableRange=Wo(e,this.max)-this._minPos,super.initOffsets(t)}buildLookupTable(t){const{min:e,max:i}=this,s=[],n=[];let o,a,r,l,h;for(o=0,a=t.length;o=e&&l<=i&&s.push(l);if(s.length<2)return[{time:e,pos:0},{time:i,pos:1}];for(o=0,a=s.length;ot.replace("rgb(","rgba(").replace(")",", 0.5)")));function Yo(t){return jo[t%jo.length]}function Uo(t){return $o[t%$o.length]}function Xo(t){let e=0;return(i,s)=>{const n=t.getDatasetMeta(s).controller;n instanceof Hn?e=function(t,e){return t.backgroundColor=t.data.map((()=>Yo(e++))),e}(i,e):n instanceof jn?e=function(t,e){return t.backgroundColor=t.data.map((()=>Uo(e++))),e}(i,e):n&&(e=function(t,e){return t.borderColor=Yo(e),t.backgroundColor=Uo(e),++e}(i,e))}}function qo(t){let e;for(e in t)if(t[e].borderColor||t[e].backgroundColor)return!0;return!1}var Ko={id:"colors",defaults:{enabled:!0,forceOverride:!1},beforeLayout(t,e,i){if(!i.enabled)return;const{data:{datasets:s},options:n}=t.config,{elements:o}=n;if(!i.forceOverride&&(qo(s)||(a=n)&&(a.borderColor||a.backgroundColor)||o&&qo(o)))return;var a;const r=Xo(t);s.forEach(r)}};function Go(t){if(t._decimated){const e=t._data;delete t._decimated,delete t._data,Object.defineProperty(t,"data",{configurable:!0,enumerable:!0,writable:!0,value:e})}}function Zo(t){t.data.datasets.forEach((t=>{Go(t)}))}var Jo={id:"decimation",defaults:{algorithm:"min-max",enabled:!1},beforeElementsUpdate:(t,e,i)=>{if(!i.enabled)return void Zo(t);const n=t.width;t.data.datasets.forEach(((e,o)=>{const{_data:a,indexAxis:r}=e,l=t.getDatasetMeta(o),h=a||e.data;if("y"===Pi([r,t.options.indexAxis]))return;if(!l.controller.supportsDecimation)return;const c=t.scales[l.xAxisID];if("linear"!==c.type&&"time"!==c.type)return;if(t.options.parsing)return;let{start:d,count:u}=function(t,e){const i=e.length;let s,n=0;const{iScale:o}=t,{min:a,max:r,minDefined:l,maxDefined:h}=o.getUserBounds();return l&&(n=J(it(e,o.axis,a).lo,0,i-1)),s=h?J(it(e,o.axis,r).hi+1,n,i)-n:i-n,{start:n,count:s}}(l,h);if(u<=(i.threshold||4*n))return void Go(e);let f;switch(s(a)&&(e._data=h,delete e.data,Object.defineProperty(e,"data",{configurable:!0,enumerable:!0,get:function(){return this._decimated},set:function(t){this._data=t}})),i.algorithm){case"lttb":f=function(t,e,i,s,n){const o=n.samples||s;if(o>=i)return t.slice(e,e+i);const a=[],r=(i-2)/(o-2);let l=0;const h=e+i-1;let c,d,u,f,g,p=e;for(a[l++]=t[p],c=0;cu&&(u=f,d=t[s],g=s);a[l++]=d,p=g}return a[l++]=t[h],a}(h,d,u,n,i);break;case"min-max":f=function(t,e,i,n){let o,a,r,l,h,c,d,u,f,g,p=0,m=0;const b=[],x=e+i-1,_=t[e].x,y=t[x].x-_;for(o=e;og&&(g=l,d=o),p=(m*p+a.x)/++m;else{const i=o-1;if(!s(c)&&!s(d)){const e=Math.min(c,d),s=Math.max(c,d);e!==u&&e!==i&&b.push({...t[e],x:p}),s!==u&&s!==i&&b.push({...t[s],x:p})}o>0&&i!==u&&b.push(t[i]),b.push(a),h=e,m=0,f=g=l,c=d=u=o}}return b}(h,d,u,n);break;default:throw new Error(`Unsupported decimation algorithm '${i.algorithm}'`)}e._decimated=f}))},destroy(t){Zo(t)}};function Qo(t,e,i,s){if(s)return;let n=e[t],o=i[t];return"angle"===t&&(n=G(n),o=G(o)),{property:t,start:n,end:o}}function ta(t,e,i){for(;e>t;e--){const t=i[e];if(!isNaN(t.x)&&!isNaN(t.y))break}return e}function ea(t,e,i,s){return t&&e?s(t[i],e[i]):t?t[i]:e?e[i]:0}function ia(t,e){let i=[],s=!1;return n(t)?(s=!0,i=t):i=function(t,e){const{x:i=null,y:s=null}=t||{},n=e.points,o=[];return e.segments.forEach((({start:t,end:e})=>{e=ta(t,e,n);const a=n[t],r=n[e];null!==s?(o.push({x:a.x,y:s}),o.push({x:r.x,y:s})):null!==i&&(o.push({x:i,y:a.y}),o.push({x:i,y:r.y}))})),o}(t,e),i.length?new so({points:i,options:{tension:0},_loop:s,_fullLoop:s}):null}function sa(t){return t&&!1!==t.fill}function na(t,e,i){let s=t[e].fill;const n=[e];let o;if(!i)return s;for(;!1!==s&&-1===n.indexOf(s);){if(!a(s))return s;if(o=t[s],!o)return!1;if(o.visible)return s;n.push(s),s=o.fill}return!1}function oa(t,e,i){const s=function(t){const e=t.options,i=e.fill;let s=l(i&&i.target,i);void 0===s&&(s=!!e.backgroundColor);if(!1===s||null===s)return!1;if(!0===s)return"origin";return s}(t);if(o(s))return!isNaN(s.value)&&s;let n=parseFloat(s);return a(n)&&Math.floor(n)===n?function(t,e,i,s){"-"!==t&&"+"!==t||(i=e+i);if(i===e||i<0||i>=s)return!1;return i}(s[0],e,n,i):["origin","start","end","stack","shape"].indexOf(s)>=0&&s}function aa(t,e,i){const s=[];for(let n=0;n=0;--e){const i=n[e].$filler;i&&(i.line.updateControlPoints(o,i.axis),s&&i.fill&&ca(t.ctx,i,o))}},beforeDatasetsDraw(t,e,i){if("beforeDatasetsDraw"!==i.drawTime)return;const s=t.getSortedVisibleDatasetMetas();for(let e=s.length-1;e>=0;--e){const i=s[e].$filler;sa(i)&&ca(t.ctx,i,t.chartArea)}},beforeDatasetDraw(t,e,i){const s=e.meta.$filler;sa(s)&&"beforeDatasetDraw"===i.drawTime&&ca(t.ctx,s,t.chartArea)},defaults:{propagate:!0,drawTime:"beforeDatasetDraw"}};const ma=(t,e)=>{let{boxHeight:i=e,boxWidth:s=e}=t;return t.usePointStyle&&(i=Math.min(i,e),s=t.pointStyleWidth||Math.min(s,e)),{boxWidth:s,boxHeight:i,itemHeight:Math.max(e,i)}};class ba extends Hs{constructor(t){super(),this._added=!1,this.legendHitBoxes=[],this._hoveredItem=null,this.doughnutMode=!1,this.chart=t.chart,this.options=t.options,this.ctx=t.ctx,this.legendItems=void 0,this.columnSizes=void 0,this.lineWidths=void 0,this.maxHeight=void 0,this.maxWidth=void 0,this.top=void 0,this.bottom=void 0,this.left=void 0,this.right=void 0,this.height=void 0,this.width=void 0,this._margins=void 0,this.position=void 0,this.weight=void 0,this.fullSize=void 0}update(t,e,i){this.maxWidth=t,this.maxHeight=e,this._margins=i,this.setDimensions(),this.buildLabels(),this.fit()}setDimensions(){this.isHorizontal()?(this.width=this.maxWidth,this.left=this._margins.left,this.right=this.width):(this.height=this.maxHeight,this.top=this._margins.top,this.bottom=this.height)}buildLabels(){const t=this.options.labels||{};let e=d(t.generateLabels,[this.chart],this)||[];t.filter&&(e=e.filter((e=>t.filter(e,this.chart.data)))),t.sort&&(e=e.sort(((e,i)=>t.sort(e,i,this.chart.data)))),this.options.reverse&&e.reverse(),this.legendItems=e}fit(){const{options:t,ctx:e}=this;if(!t.display)return void(this.width=this.height=0);const i=t.labels,s=Si(i.font),n=s.size,o=this._computeTitleHeight(),{boxWidth:a,itemHeight:r}=ma(i,n);let l,h;e.font=s.string,this.isHorizontal()?(l=this.maxWidth,h=this._fitRows(o,n,a,r)+10):(h=this.maxHeight,l=this._fitCols(o,s,a,r)+10),this.width=Math.min(l,t.maxWidth||this.maxWidth),this.height=Math.min(h,t.maxHeight||this.maxHeight)}_fitRows(t,e,i,s){const{ctx:n,maxWidth:o,options:{labels:{padding:a}}}=this,r=this.legendHitBoxes=[],l=this.lineWidths=[0],h=s+a;let c=t;n.textAlign="left",n.textBaseline="middle";let d=-1,u=-h;return this.legendItems.forEach(((t,f)=>{const g=i+e/2+n.measureText(t.text).width;(0===f||l[l.length-1]+g+2*a>o)&&(c+=h,l[l.length-(f>0?0:1)]=0,u+=h,d++),r[f]={left:0,top:u,row:d,width:g,height:s},l[l.length-1]+=g+a})),c}_fitCols(t,e,i,s){const{ctx:n,maxHeight:o,options:{labels:{padding:a}}}=this,r=this.legendHitBoxes=[],l=this.columnSizes=[],h=o-t;let c=a,d=0,u=0,f=0,g=0;return this.legendItems.forEach(((t,o)=>{const{itemWidth:p,itemHeight:m}=function(t,e,i,s,n){const o=function(t,e,i,s){let n=t.text;n&&"string"!=typeof n&&(n=n.reduce(((t,e)=>t.length>e.length?t:e)));return e+i.size/2+s.measureText(n).width}(s,t,e,i),a=function(t,e,i){let s=t;"string"!=typeof e.text&&(s=xa(e,i));return s}(n,s,e.lineHeight);return{itemWidth:o,itemHeight:a}}(i,e,n,t,s);o>0&&u+m+2*a>h&&(c+=d+a,l.push({width:d,height:u}),f+=d+a,g++,d=u=0),r[o]={left:f,top:u,col:g,width:p,height:m},d=Math.max(d,p),u+=m+a})),c+=d,l.push({width:d,height:u}),c}adjustHitBoxes(){if(!this.options.display)return;const t=this._computeTitleHeight(),{legendHitBoxes:e,options:{align:i,labels:{padding:s},rtl:n}}=this,o=Oi(n,this.left,this.width);if(this.isHorizontal()){let n=0,a=ft(i,this.left+s,this.right-this.lineWidths[n]);for(const r of e)n!==r.row&&(n=r.row,a=ft(i,this.left+s,this.right-this.lineWidths[n])),r.top+=this.top+t+s,r.left=o.leftForLtr(o.x(a),r.width),a+=r.width+s}else{let n=0,a=ft(i,this.top+t+s,this.bottom-this.columnSizes[n].height);for(const r of e)r.col!==n&&(n=r.col,a=ft(i,this.top+t+s,this.bottom-this.columnSizes[n].height)),r.top=a,r.left+=this.left+s,r.left=o.leftForLtr(o.x(r.left),r.width),a+=r.height+s}}isHorizontal(){return"top"===this.options.position||"bottom"===this.options.position}draw(){if(this.options.display){const t=this.ctx;Ie(t,this),this._draw(),ze(t)}}_draw(){const{options:t,columnSizes:e,lineWidths:i,ctx:s}=this,{align:n,labels:o}=t,a=ue.color,r=Oi(t.rtl,this.left,this.width),h=Si(o.font),{padding:c}=o,d=h.size,u=d/2;let f;this.drawTitle(),s.textAlign=r.textAlign("left"),s.textBaseline="middle",s.lineWidth=.5,s.font=h.string;const{boxWidth:g,boxHeight:p,itemHeight:m}=ma(o,d),b=this.isHorizontal(),x=this._computeTitleHeight();f=b?{x:ft(n,this.left+c,this.right-i[0]),y:this.top+c+x,line:0}:{x:this.left+c,y:ft(n,this.top+x+c,this.bottom-e[0].height),line:0},Ai(this.ctx,t.textDirection);const _=m+c;this.legendItems.forEach(((y,v)=>{s.strokeStyle=y.fontColor,s.fillStyle=y.fontColor;const M=s.measureText(y.text).width,w=r.textAlign(y.textAlign||(y.textAlign=o.textAlign)),k=g+u+M;let S=f.x,P=f.y;r.setWidth(this.width),b?v>0&&S+k+c>this.right&&(P=f.y+=_,f.line++,S=f.x=ft(n,this.left+c,this.right-i[f.line])):v>0&&P+_>this.bottom&&(S=f.x=S+e[f.line].width+c,f.line++,P=f.y=ft(n,this.top+x+c,this.bottom-e[f.line].height));if(function(t,e,i){if(isNaN(g)||g<=0||isNaN(p)||p<0)return;s.save();const n=l(i.lineWidth,1);if(s.fillStyle=l(i.fillStyle,a),s.lineCap=l(i.lineCap,"butt"),s.lineDashOffset=l(i.lineDashOffset,0),s.lineJoin=l(i.lineJoin,"miter"),s.lineWidth=n,s.strokeStyle=l(i.strokeStyle,a),s.setLineDash(l(i.lineDash,[])),o.usePointStyle){const a={radius:p*Math.SQRT2/2,pointStyle:i.pointStyle,rotation:i.rotation,borderWidth:n},l=r.xPlus(t,g/2);Ee(s,a,l,e+u,o.pointStyleWidth&&g)}else{const o=e+Math.max((d-p)/2,0),a=r.leftForLtr(t,g),l=wi(i.borderRadius);s.beginPath(),Object.values(l).some((t=>0!==t))?He(s,{x:a,y:o,w:g,h:p,radius:l}):s.rect(a,o,g,p),s.fill(),0!==n&&s.stroke()}s.restore()}(r.x(S),P,y),S=gt(w,S+g+u,b?S+k:this.right,t.rtl),function(t,e,i){We(s,i.text,t,e+m/2,h,{strikethrough:i.hidden,textAlign:r.textAlign(i.textAlign)})}(r.x(S),P,y),b)f.x+=k+c;else if("string"!=typeof y.text){const t=h.lineHeight;f.y+=xa(y,t)}else f.y+=_})),Ti(this.ctx,t.textDirection)}drawTitle(){const t=this.options,e=t.title,i=Si(e.font),s=ki(e.padding);if(!e.display)return;const n=Oi(t.rtl,this.left,this.width),o=this.ctx,a=e.position,r=i.size/2,l=s.top+r;let h,c=this.left,d=this.width;if(this.isHorizontal())d=Math.max(...this.lineWidths),h=this.top+l,c=ft(t.align,c,this.right-d);else{const e=this.columnSizes.reduce(((t,e)=>Math.max(t,e.height)),0);h=l+ft(t.align,this.top,this.bottom-e-t.labels.padding-this._computeTitleHeight())}const u=ft(a,c,c+d);o.textAlign=n.textAlign(ut(a)),o.textBaseline="middle",o.strokeStyle=e.color,o.fillStyle=e.color,o.font=i.string,We(o,e.text,u,h,i)}_computeTitleHeight(){const t=this.options.title,e=Si(t.font),i=ki(t.padding);return t.display?e.lineHeight+i.height:0}_getLegendItemAt(t,e){let i,s,n;if(tt(t,this.left,this.right)&&tt(e,this.top,this.bottom))for(n=this.legendHitBoxes,i=0;it.chart.options.color,boxWidth:40,padding:10,generateLabels(t){const e=t.data.datasets,{labels:{usePointStyle:i,pointStyle:s,textAlign:n,color:o,useBorderRadius:a,borderRadius:r}}=t.legend.options;return t._getSortedDatasetMetas().map((t=>{const l=t.controller.getStyle(i?0:void 0),h=ki(l.borderWidth);return{text:e[t.index].label,fillStyle:l.backgroundColor,fontColor:o,hidden:!t.visible,lineCap:l.borderCapStyle,lineDash:l.borderDash,lineDashOffset:l.borderDashOffset,lineJoin:l.borderJoinStyle,lineWidth:(h.width+h.height)/4,strokeStyle:l.borderColor,pointStyle:s||l.pointStyle,rotation:l.rotation,textAlign:n||l.textAlign,borderRadius:a&&(r||l.borderRadius),datasetIndex:t.index}}),this)}},title:{color:t=>t.chart.options.color,display:!1,position:"center",text:""}},descriptors:{_scriptable:t=>!t.startsWith("on"),labels:{_scriptable:t=>!["generateLabels","filter","sort"].includes(t)}}};class ya extends Hs{constructor(t){super(),this.chart=t.chart,this.options=t.options,this.ctx=t.ctx,this._padding=void 0,this.top=void 0,this.bottom=void 0,this.left=void 0,this.right=void 0,this.width=void 0,this.height=void 0,this.position=void 0,this.weight=void 0,this.fullSize=void 0}update(t,e){const i=this.options;if(this.left=0,this.top=0,!i.display)return void(this.width=this.height=this.right=this.bottom=0);this.width=this.right=t,this.height=this.bottom=e;const s=n(i.text)?i.text.length:1;this._padding=ki(i.padding);const o=s*Si(i.font).lineHeight+this._padding.height;this.isHorizontal()?this.height=o:this.width=o}isHorizontal(){const t=this.options.position;return"top"===t||"bottom"===t}_drawArgs(t){const{top:e,left:i,bottom:s,right:n,options:o}=this,a=o.align;let r,l,h,c=0;return this.isHorizontal()?(l=ft(a,i,n),h=e+t,r=n-i):("left"===o.position?(l=i+t,h=ft(a,s,e),c=-.5*C):(l=n-t,h=ft(a,e,s),c=.5*C),r=s-e),{titleX:l,titleY:h,maxWidth:r,rotation:c}}draw(){const t=this.ctx,e=this.options;if(!e.display)return;const i=Si(e.font),s=i.lineHeight/2+this._padding.top,{titleX:n,titleY:o,maxWidth:a,rotation:r}=this._drawArgs(s);We(t,e.text,0,0,i,{color:e.color,maxWidth:a,rotation:r,textAlign:ut(e.align),textBaseline:"middle",translation:[n,o]})}}var va={id:"title",_element:ya,start(t,e,i){!function(t,e){const i=new ya({ctx:t.ctx,options:e,chart:t});as.configure(t,i,e),as.addBox(t,i),t.titleBlock=i}(t,i)},stop(t){const e=t.titleBlock;as.removeBox(t,e),delete t.titleBlock},beforeUpdate(t,e,i){const s=t.titleBlock;as.configure(t,s,i),s.options=i},defaults:{align:"center",display:!1,font:{weight:"bold"},fullSize:!0,padding:10,position:"top",text:"",weight:2e3},defaultRoutes:{color:"color"},descriptors:{_scriptable:!0,_indexable:!1}};const Ma=new WeakMap;var wa={id:"subtitle",start(t,e,i){const s=new ya({ctx:t.ctx,options:i,chart:t});as.configure(t,s,i),as.addBox(t,s),Ma.set(t,s)},stop(t){as.removeBox(t,Ma.get(t)),Ma.delete(t)},beforeUpdate(t,e,i){const s=Ma.get(t);as.configure(t,s,i),s.options=i},defaults:{align:"center",display:!1,font:{weight:"normal"},fullSize:!0,padding:0,position:"top",text:"",weight:1500},defaultRoutes:{color:"color"},descriptors:{_scriptable:!0,_indexable:!1}};const ka={average(t){if(!t.length)return!1;let e,i,s=0,n=0,o=0;for(e=0,i=t.length;e-1?t.split("\n"):t}function Da(t,e){const{element:i,datasetIndex:s,index:n}=e,o=t.getDatasetMeta(s).controller,{label:a,value:r}=o.getLabelAndValue(n);return{chart:t,label:a,parsed:o.getParsed(n),raw:t.data.datasets[s].data[n],formattedValue:r,dataset:o.getDataset(),dataIndex:n,datasetIndex:s,element:i}}function Ca(t,e){const i=t.chart.ctx,{body:s,footer:n,title:o}=t,{boxWidth:a,boxHeight:r}=e,l=Si(e.bodyFont),h=Si(e.titleFont),c=Si(e.footerFont),d=o.length,f=n.length,g=s.length,p=ki(e.padding);let m=p.height,b=0,x=s.reduce(((t,e)=>t+e.before.length+e.lines.length+e.after.length),0);if(x+=t.beforeBody.length+t.afterBody.length,d&&(m+=d*h.lineHeight+(d-1)*e.titleSpacing+e.titleMarginBottom),x){m+=g*(e.displayColors?Math.max(r,l.lineHeight):l.lineHeight)+(x-g)*l.lineHeight+(x-1)*e.bodySpacing}f&&(m+=e.footerMarginTop+f*c.lineHeight+(f-1)*e.footerSpacing);let _=0;const y=function(t){b=Math.max(b,i.measureText(t).width+_)};return i.save(),i.font=h.string,u(t.title,y),i.font=l.string,u(t.beforeBody.concat(t.afterBody),y),_=e.displayColors?a+2+e.boxPadding:0,u(s,(t=>{u(t.before,y),u(t.lines,y),u(t.after,y)})),_=0,i.font=c.string,u(t.footer,y),i.restore(),b+=p.width,{width:b,height:m}}function Oa(t,e,i,s){const{x:n,width:o}=i,{width:a,chartArea:{left:r,right:l}}=t;let h="center";return"center"===s?h=n<=(r+l)/2?"left":"right":n<=o/2?h="left":n>=a-o/2&&(h="right"),function(t,e,i,s){const{x:n,width:o}=s,a=i.caretSize+i.caretPadding;return"left"===t&&n+o+a>e.width||"right"===t&&n-o-a<0||void 0}(h,t,e,i)&&(h="center"),h}function Aa(t,e,i){const s=i.yAlign||e.yAlign||function(t,e){const{y:i,height:s}=e;return it.height-s/2?"bottom":"center"}(t,i);return{xAlign:i.xAlign||e.xAlign||Oa(t,e,i,s),yAlign:s}}function Ta(t,e,i,s){const{caretSize:n,caretPadding:o,cornerRadius:a}=t,{xAlign:r,yAlign:l}=i,h=n+o,{topLeft:c,topRight:d,bottomLeft:u,bottomRight:f}=wi(a);let g=function(t,e){let{x:i,width:s}=t;return"right"===e?i-=s:"center"===e&&(i-=s/2),i}(e,r);const p=function(t,e,i){let{y:s,height:n}=t;return"top"===e?s+=i:s-="bottom"===e?n+i:n/2,s}(e,l,h);return"center"===l?"left"===r?g+=h:"right"===r&&(g-=h):"left"===r?g-=Math.max(c,u)+n:"right"===r&&(g+=Math.max(d,f)+n),{x:J(g,0,s.width-e.width),y:J(p,0,s.height-e.height)}}function La(t,e,i){const s=ki(i.padding);return"center"===e?t.x+t.width/2:"right"===e?t.x+t.width-s.right:t.x+s.left}function Ea(t){return Sa([],Pa(t))}function Ra(t,e){const i=e&&e.dataset&&e.dataset.tooltip&&e.dataset.tooltip.callbacks;return i?t.override(i):t}const Ia={beforeTitle:e,title(t){if(t.length>0){const e=t[0],i=e.chart.data.labels,s=i?i.length:0;if(this&&this.options&&"dataset"===this.options.mode)return e.dataset.label||"";if(e.label)return e.label;if(s>0&&e.dataIndex{const e={before:[],lines:[],after:[]},n=Ra(i,t);Sa(e.before,Pa(za(n,"beforeLabel",this,t))),Sa(e.lines,za(n,"label",this,t)),Sa(e.after,Pa(za(n,"afterLabel",this,t))),s.push(e)})),s}getAfterBody(t,e){return Ea(za(e.callbacks,"afterBody",this,t))}getFooter(t,e){const{callbacks:i}=e,s=za(i,"beforeFooter",this,t),n=za(i,"footer",this,t),o=za(i,"afterFooter",this,t);let a=[];return a=Sa(a,Pa(s)),a=Sa(a,Pa(n)),a=Sa(a,Pa(o)),a}_createItems(t){const e=this._active,i=this.chart.data,s=[],n=[],o=[];let a,r,l=[];for(a=0,r=e.length;at.filter(e,s,n,i)))),t.itemSort&&(l=l.sort(((e,s)=>t.itemSort(e,s,i)))),u(l,(e=>{const i=Ra(t.callbacks,e);s.push(za(i,"labelColor",this,e)),n.push(za(i,"labelPointStyle",this,e)),o.push(za(i,"labelTextColor",this,e))})),this.labelColors=s,this.labelPointStyles=n,this.labelTextColors=o,this.dataPoints=l,l}update(t,e){const i=this.options.setContext(this.getContext()),s=this._active;let n,o=[];if(s.length){const t=ka[i.position].call(this,s,this._eventPosition);o=this._createItems(i),this.title=this.getTitle(o,i),this.beforeBody=this.getBeforeBody(o,i),this.body=this.getBody(o,i),this.afterBody=this.getAfterBody(o,i),this.footer=this.getFooter(o,i);const e=this._size=Ca(this,i),a=Object.assign({},t,e),r=Aa(this.chart,i,a),l=Ta(i,a,r,this.chart);this.xAlign=r.xAlign,this.yAlign=r.yAlign,n={opacity:1,x:l.x,y:l.y,width:e.width,height:e.height,caretX:t.x,caretY:t.y}}else 0!==this.opacity&&(n={opacity:0});this._tooltipItems=o,this.$context=void 0,n&&this._resolveAnimations().update(this,n),t&&i.external&&i.external.call(this,{chart:this.chart,tooltip:this,replay:e})}drawCaret(t,e,i,s){const n=this.getCaretPosition(t,i,s);e.lineTo(n.x1,n.y1),e.lineTo(n.x2,n.y2),e.lineTo(n.x3,n.y3)}getCaretPosition(t,e,i){const{xAlign:s,yAlign:n}=this,{caretSize:o,cornerRadius:a}=i,{topLeft:r,topRight:l,bottomLeft:h,bottomRight:c}=wi(a),{x:d,y:u}=t,{width:f,height:g}=e;let p,m,b,x,_,y;return"center"===n?(_=u+g/2,"left"===s?(p=d,m=p-o,x=_+o,y=_-o):(p=d+f,m=p+o,x=_-o,y=_+o),b=p):(m="left"===s?d+Math.max(r,h)+o:"right"===s?d+f-Math.max(l,c)-o:this.caretX,"top"===n?(x=u,_=x-o,p=m-o,b=m+o):(x=u+g,_=x+o,p=m+o,b=m-o),y=x),{x1:p,x2:m,x3:b,y1:x,y2:_,y3:y}}drawTitle(t,e,i){const s=this.title,n=s.length;let o,a,r;if(n){const l=Oi(i.rtl,this.x,this.width);for(t.x=La(this,i.titleAlign,i),e.textAlign=l.textAlign(i.titleAlign),e.textBaseline="middle",o=Si(i.titleFont),a=i.titleSpacing,e.fillStyle=i.titleColor,e.font=o.string,r=0;r0!==t))?(t.beginPath(),t.fillStyle=n.multiKeyBackground,He(t,{x:e,y:g,w:h,h:l,radius:r}),t.fill(),t.stroke(),t.fillStyle=a.backgroundColor,t.beginPath(),He(t,{x:i,y:g+1,w:h-2,h:l-2,radius:r}),t.fill()):(t.fillStyle=n.multiKeyBackground,t.fillRect(e,g,h,l),t.strokeRect(e,g,h,l),t.fillStyle=a.backgroundColor,t.fillRect(i,g+1,h-2,l-2))}t.fillStyle=this.labelTextColors[i]}drawBody(t,e,i){const{body:s}=this,{bodySpacing:n,bodyAlign:o,displayColors:a,boxHeight:r,boxWidth:l,boxPadding:h}=i,c=Si(i.bodyFont);let d=c.lineHeight,f=0;const g=Oi(i.rtl,this.x,this.width),p=function(i){e.fillText(i,g.x(t.x+f),t.y+d/2),t.y+=d+n},m=g.textAlign(o);let b,x,_,y,v,M,w;for(e.textAlign=o,e.textBaseline="middle",e.font=c.string,t.x=La(this,m,i),e.fillStyle=i.bodyColor,u(this.beforeBody,p),f=a&&"right"!==m?"center"===o?l/2+h:l+2+h:0,y=0,M=s.length;y0&&e.stroke()}_updateAnimationTarget(t){const e=this.chart,i=this.$animations,s=i&&i.x,n=i&&i.y;if(s||n){const i=ka[t.position].call(this,this._active,this._eventPosition);if(!i)return;const o=this._size=Ca(this,t),a=Object.assign({},i,this._size),r=Aa(e,t,a),l=Ta(t,a,r,e);s._to===l.x&&n._to===l.y||(this.xAlign=r.xAlign,this.yAlign=r.yAlign,this.width=o.width,this.height=o.height,this.caretX=i.x,this.caretY=i.y,this._resolveAnimations().update(this,l))}}_willRender(){return!!this.opacity}draw(t){const e=this.options.setContext(this.getContext());let i=this.opacity;if(!i)return;this._updateAnimationTarget(e);const s={width:this.width,height:this.height},n={x:this.x,y:this.y};i=Math.abs(i)<.001?0:i;const o=ki(e.padding),a=this.title.length||this.beforeBody.length||this.body.length||this.afterBody.length||this.footer.length;e.enabled&&a&&(t.save(),t.globalAlpha=i,this.drawBackground(n,t,s,e),Ai(t,e.textDirection),n.y+=o.top,this.drawTitle(n,t,e),this.drawBody(n,t,e),this.drawFooter(n,t,e),Ti(t,e.textDirection),t.restore())}getActiveElements(){return this._active||[]}setActiveElements(t,e){const i=this._active,s=t.map((({datasetIndex:t,index:e})=>{const i=this.chart.getDatasetMeta(t);if(!i)throw new Error("Cannot find a dataset at index "+t);return{datasetIndex:t,element:i.data[e],index:e}})),n=!f(i,s),o=this._positionChanged(s,e);(n||o)&&(this._active=s,this._eventPosition=e,this._ignoreReplayEvents=!0,this.update(!0))}handleEvent(t,e,i=!0){if(e&&this._ignoreReplayEvents)return!1;this._ignoreReplayEvents=!1;const s=this.options,n=this._active||[],o=this._getActiveElements(t,n,e,i),a=this._positionChanged(o,t),r=e||!f(o,n)||a;return r&&(this._active=o,(s.enabled||s.external)&&(this._eventPosition={x:t.x,y:t.y},this.update(!0,e))),r}_getActiveElements(t,e,i,s){const n=this.options;if("mouseout"===t.type)return[];if(!s)return e;const o=this.chart.getElementsAtEventForMode(t,n.mode,n,i);return n.reverse&&o.reverse(),o}_positionChanged(t,e){const{caretX:i,caretY:s,options:n}=this,o=ka[n.position].call(this,t,e);return!1!==o&&(i!==o.x||s!==o.y)}}var Va={id:"tooltip",_element:Fa,positioners:ka,afterInit(t,e,i){i&&(t.tooltip=new Fa({chart:t,options:i}))},beforeUpdate(t,e,i){t.tooltip&&t.tooltip.initialize(i)},reset(t,e,i){t.tooltip&&t.tooltip.initialize(i)},afterDraw(t){const e=t.tooltip;if(e&&e._willRender()){const i={tooltip:e};if(!1===t.notifyPlugins("beforeTooltipDraw",{...i,cancelable:!0}))return;e.draw(t.ctx),t.notifyPlugins("afterTooltipDraw",i)}},afterEvent(t,e){if(t.tooltip){const i=e.replay;t.tooltip.handleEvent(e.event,i,e.inChartArea)&&(e.changed=!0)}},defaults:{enabled:!0,external:null,position:"average",backgroundColor:"rgba(0,0,0,0.8)",titleColor:"#fff",titleFont:{weight:"bold"},titleSpacing:2,titleMarginBottom:6,titleAlign:"left",bodyColor:"#fff",bodySpacing:2,bodyFont:{},bodyAlign:"left",footerColor:"#fff",footerSpacing:2,footerMarginTop:6,footerFont:{weight:"bold"},footerAlign:"left",padding:6,caretPadding:2,caretSize:5,cornerRadius:6,boxHeight:(t,e)=>e.bodyFont.size,boxWidth:(t,e)=>e.bodyFont.size,multiKeyBackground:"#fff",displayColors:!0,boxPadding:0,borderColor:"rgba(0,0,0,0)",borderWidth:0,animation:{duration:400,easing:"easeOutQuart"},animations:{numbers:{type:"number",properties:["x","y","width","height","caretX","caretY"]},opacity:{easing:"linear",duration:200}},callbacks:Ia},defaultRoutes:{bodyFont:"font",footerFont:"font",titleFont:"font"},descriptors:{_scriptable:t=>"filter"!==t&&"itemSort"!==t&&"external"!==t,_indexable:!1,callbacks:{_scriptable:!1,_indexable:!1},animation:{_fallback:!1},animations:{_fallback:"animation"}},additionalOptionScopes:["interaction"]};return On.register($n,Ho,uo,t),On.helpers={...Ni},On._adapters=En,On.Animation=Cs,On.Animations=Os,On.animator=xt,On.controllers=en.controllers.items,On.DatasetController=Ws,On.Element=Hs,On.elements=uo,On.Interaction=Xi,On.layouts=as,On.platforms=Ss,On.Scale=Js,On.Ticks=ae,Object.assign(On,$n,Ho,uo,t,Ss),On.Chart=On,"undefined"!=typeof window&&(window.Chart=On),On})); +//# sourceMappingURL=chart.umd.js.map \ No newline at end of file diff --git a/coreplugins/projects-charts/templates/index.html b/coreplugins/projects-charts/templates/index.html new file mode 100644 index 000000000..b2182265d --- /dev/null +++ b/coreplugins/projects-charts/templates/index.html @@ -0,0 +1,126 @@ +{% extends "app/plugins/templates/base.html" %} +{% load i18n %} +{% block content %} + +
+
+

{% trans 'Projects' %}

+
+
+
+ +
+ {% for field in form %} +
+ + {{ field }} + {% endfor %} +
+ + +
+ {{ form.non_field_errors }} + +
+
+
+ +
+
+ +
+
+ +
+ +
+ +
+
+ +{% endblock %} \ No newline at end of file From ce108ec119841c79c341a15ea2ffb9a106db032e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sun, 18 Jun 2023 16:57:13 +0200 Subject: [PATCH 03/95] Add delete button for read only projects --- app/api/projects.py | 4 +++- .../app/js/components/ProjectListItem.jsx | 20 +++++++++++++++++++ app/tests/test_api.py | 15 ++++++++------ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/app/api/projects.py b/app/api/projects.py index 308198da5..b5d932d40 100644 --- a/app/api/projects.py +++ b/app/api/projects.py @@ -197,10 +197,12 @@ def edit(self, request, pk=None): return Response({'success': True}, status=status.HTTP_200_OK) def destroy(self, request, pk=None): - project = get_and_check_project(request, pk, ('delete_project', )) + project = get_and_check_project(request, pk, ('view_project', )) # Owner? Delete the project if project.owner == request.user or request.user.is_superuser: + get_and_check_project(request, pk, ('delete_project', )) + return super().destroy(self, request, pk=pk) else: # Do not remove the project, simply remove all user's permissions to the project diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 7323b0845..6588b8efe 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -400,6 +400,20 @@ class ProjectListItem extends React.Component { this.editProjectDialog.show(); } + handleHideProject = (deleteWarning, deleteAction) => { + return () => { + if (window.confirm(deleteWarning)){ + this.setState({error: "", refreshing: true}); + deleteAction() + .fail(e => { + this.setState({error: e.message || (e.responseJSON || {}).detail || e.responseText || _("Could not delete item")}); + }).always(() => { + this.setState({refreshing: false}); + }); + } + } + } + updateProject(project){ return $.ajax({ url: `/api/projects/${this.state.data.id}/edit/`, @@ -683,6 +697,12 @@ class ProjectListItem extends React.Component { ] : ""} + {!canEdit && !data.owned ? + [ + , {_("Delete")} + ] + : ""} + diff --git a/app/tests/test_api.py b/app/tests/test_api.py index bdbc9d819..fd392a35f 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -253,14 +253,17 @@ def test_projects_and_tasks(self): for perm in ['delete', 'change', 'add']: self.assertFalse(perm in res.data['permissions']) - # Can't delete a project for which we just have view permissions - res = client.delete('/api/projects/{}/'.format(other_temp_project.id)) - self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) - - # Can delete a project for which we have delete permissions - assign_perm('delete_project', user, other_temp_project) + # Can delete a project for which we just have view permissions + # (we will just remove our read permissions without deleting the project) res = client.delete('/api/projects/{}/'.format(other_temp_project.id)) self.assertTrue(res.status_code == status.HTTP_204_NO_CONTENT) + + # Project still exists + self.assertTrue(Project.objects.filter(id=other_temp_project.id).count() == 1) + + # We just can't access it + res = client.get('/api/projects/{}/'.format(other_temp_project.id)) + self.assertTrue(res.status_code == status.HTTP_404_NOT_FOUND) # A user cannot reassign a task to a # project for which he/she has no permissions From 7e9791d5c1d103d98f42a39f08bef431fd7b76b4 Mon Sep 17 00:00:00 2001 From: vinsonliux <79687417+vinsonliux@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:08:56 +0800 Subject: [PATCH 04/95] Fixed the uploaded file was too large, set maxFilesize = 128G --- app/static/app/js/components/ProjectListItem.jsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 6588b8efe..37e4fd3e1 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -143,6 +143,7 @@ class ProjectListItem extends React.Component { autoProcessQueue: false, createImageThumbnails: false, clickable: this.uploadButton, + maxFilesize: 131072, // 128G chunkSize: 2147483647, timeout: 2147483647, @@ -215,6 +216,14 @@ class ProjectListItem extends React.Component { try{ if (file.status === "error"){ + if ((file.size / 1024) > this.dz.options.maxFilesize) { + // Delete from upload queue + this.setUploadState({ + totalCount: this.state.upload.totalCount - 1, + totalBytes: this.state.upload.totalBytes - file.size + }); + throw new Error(interpolate(_('Cannot upload %(filename)s, File too Large! Default MaxFileSize is %(maxFileSize)s MB!'), { filename: file.name, maxFileSize: this.dz.options.maxFilesize })); + } retry(); }else{ // Check response From 4b870076829ea69d6ad1257d397300a622976709 Mon Sep 17 00:00:00 2001 From: vinsonliux <79687417+vinsonliux@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:24:14 +0800 Subject: [PATCH 05/95] format add code --- .../app/js/components/ProjectListItem.jsx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 37e4fd3e1..b4a8eb410 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -143,7 +143,7 @@ class ProjectListItem extends React.Component { autoProcessQueue: false, createImageThumbnails: false, clickable: this.uploadButton, - maxFilesize: 131072, // 128G + maxFilesize: 131072, // 128G chunkSize: 2147483647, timeout: 2147483647, @@ -216,14 +216,14 @@ class ProjectListItem extends React.Component { try{ if (file.status === "error"){ - if ((file.size / 1024) > this.dz.options.maxFilesize) { - // Delete from upload queue - this.setUploadState({ - totalCount: this.state.upload.totalCount - 1, - totalBytes: this.state.upload.totalBytes - file.size - }); - throw new Error(interpolate(_('Cannot upload %(filename)s, File too Large! Default MaxFileSize is %(maxFileSize)s MB!'), { filename: file.name, maxFileSize: this.dz.options.maxFilesize })); - } + if ((file.size / 1024) > this.dz.options.maxFilesize) { + // Delete from upload queue + this.setUploadState({ + totalCount: this.state.upload.totalCount - 1, + totalBytes: this.state.upload.totalBytes - file.size + }); + throw new Error(interpolate(_('Cannot upload %(filename)s, File too Large! Default MaxFileSize is %(maxFileSize)s MB!'), { filename: file.name, maxFileSize: this.dz.options.maxFilesize })); + } retry(); }else{ // Check response From 397117fad188628902fd320f1639f9322ef38e03 Mon Sep 17 00:00:00 2001 From: vinsonliux <79687417+vinsonliux@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:26:18 +0800 Subject: [PATCH 06/95] code format --- app/static/app/js/components/ProjectListItem.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index b4a8eb410..31d6ec429 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -223,7 +223,7 @@ class ProjectListItem extends React.Component { totalBytes: this.state.upload.totalBytes - file.size }); throw new Error(interpolate(_('Cannot upload %(filename)s, File too Large! Default MaxFileSize is %(maxFileSize)s MB!'), { filename: file.name, maxFileSize: this.dz.options.maxFilesize })); - } + } retry(); }else{ // Check response From 2035b3a3fead0a276c2bd7461f7f4bff06a3bfdf Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 14 Aug 2023 20:28:41 -0400 Subject: [PATCH 07/95] Update locale --- locale | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locale b/locale index 31a7b8fc6..f2898d595 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 31a7b8fc6d955e8bd6c13d2de84501bc43895190 +Subproject commit f2898d595d71ece8d69d998a41e8348750ef788f From 544b06a81ace896487cd49eef42011e18a654cdc Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 15 Aug 2023 15:36:52 -0400 Subject: [PATCH 08/95] Lock pydantic --- locale | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/locale b/locale index f2898d595..31a7b8fc6 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit f2898d595d71ece8d69d998a41e8348750ef788f +Subproject commit 31a7b8fc6d955e8bd6c13d2de84501bc43895190 diff --git a/requirements.txt b/requirements.txt index 1f02f3989..61aa2130b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ Pillow==8.3.2 pip-autoremove==0.9.0 psycopg2-binary==2.8.6 PyJWT==1.5.3 +pydantic==1.10.8 pyodm==1.5.10 pyparsing==2.4.7 pytz==2020.1 From 84356f1ce790d318e176fffe4e3de0907786529a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Aug 2023 11:43:50 -0400 Subject: [PATCH 09/95] External auth PoC, add task sizes --- app/api/tasks.py | 2 +- app/auth/backends.py | 73 +++++++++++++++++++ app/migrations/0036_task_size.py | 50 +++++++++++++ app/models/task.py | 14 ++++ app/static/app/js/classes/Utils.js | 10 +++ app/static/app/js/components/TaskListItem.jsx | 6 ++ .../app/js/components/UploadProgressBar.jsx | 13 +--- package.json | 2 +- webodm/settings.py | 5 ++ 9 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 app/auth/backends.py create mode 100644 app/migrations/0036_task_size.py diff --git a/app/api/tasks.py b/app/api/tasks.py index 8e4d9f2eb..4e4da2da6 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -75,7 +75,7 @@ def get_can_rerun_from(self, obj): class Meta: model = models.Task exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', ) - read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', ) + read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', ) class TaskViewSet(viewsets.ViewSet): """ diff --git a/app/auth/backends.py b/app/auth/backends.py new file mode 100644 index 000000000..95fcb2c12 --- /dev/null +++ b/app/auth/backends.py @@ -0,0 +1,73 @@ +import requests +from django.contrib.auth.backends import ModelBackend +from django.contrib.auth.models import User +from nodeodm.models import ProcessingNode +from webodm.settings import EXTERNAL_AUTH_ENDPOINT, USE_EXTERNAL_AUTH +from guardian.shortcuts import assign_perm +import logging + +logger = logging.getLogger('app.logger') + +class ExternalBackend(ModelBackend): + def authenticate(self, request, username=None, password=None): + if not USE_EXTERNAL_AUTH: + return None + + try: + r = requests.post(EXTERNAL_AUTH_ENDPOINT, { + 'username': username, + 'password': password + }, headers={'Accept': 'application/json'}) + res = r.json() + + if 'message' in res or 'error' in res: + return None + + logger.info(res) + + if 'user_id' in res: + try: + user = User.objects.get(pk=res['user_id']) + + # Update user info + if user.username != username: + user.username = username + user.save() + except User.DoesNotExist: + user = User(pk=res['user_id'], username=username) + user.save() + + # Setup/update processing node + if ('api_key' in res or 'token' in res) and 'node' in res: + hostname = res['node']['hostname'] + port = res['node']['port'] + token = res['api_key'] if 'api_key' in res else res['token'] + + try: + node = ProcessingNode.objects.get(token=token) + if node.hostname != hostname or node.port != port: + node.hostname = hostname + node.port = port + node.save() + + except ProcessingNode.DoesNotExist: + node = ProcessingNode(hostname=hostname, port=port, token=token) + node.save() + + if not user.has_perm('view_processingnode', node): + assign_perm('view_processingnode', user, node) + + return user + else: + return None + except: + return None + + def get_user(self, user_id): + if not USE_EXTERNAL_AUTH: + return None + + try: + return User.objects.get(pk=user_id) + except User.DoesNotExist: + return None \ No newline at end of file diff --git a/app/migrations/0036_task_size.py b/app/migrations/0036_task_size.py new file mode 100644 index 000000000..1fe421953 --- /dev/null +++ b/app/migrations/0036_task_size.py @@ -0,0 +1,50 @@ +# Generated by Django 2.2.27 on 2023-08-21 14:50 +import os +from django.db import migrations, models +from webodm import settings + +def task_path(project_id, task_id, *args): + return os.path.join(settings.MEDIA_ROOT, + "project", + str(project_id), + "task", + str(task_id), + *args) + +def update_size(task): + try: + total_bytes = 0 + for dirpath, _, filenames in os.walk(task_path(task.project.id, task.id)): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total_bytes += os.path.getsize(fp) + task.size = (total_bytes / 1024 / 1024) + task.save() + print("Updated {} with size {}".format(task, task.size)) + except Exception as e: + print("Cannot update size for task {}: {}".format(task, str(e))) + + + +def update_task_sizes(apps, schema_editor): + Task = apps.get_model('app', 'Task') + + for t in Task.objects.all(): + update_size(t) + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0035_task_orthophoto_bands'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='size', + field=models.FloatField(blank=True, default=0.0, help_text='Size of the task on disk in megabytes', verbose_name='Size'), + ), + + migrations.RunPython(update_task_sizes), + ] diff --git a/app/models/task.py b/app/models/task.py index be2c49876..09d51aa8a 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -279,6 +279,7 @@ class Task(models.Model): epsg = models.IntegerField(null=True, default=None, blank=True, help_text=_("EPSG code of the dataset (if georeferenced)"), verbose_name="EPSG") tags = models.TextField(db_index=True, default="", blank=True, help_text=_("Task tags"), verbose_name=_("Tags")) orthophoto_bands = fields.JSONField(default=list, blank=True, help_text=_("List of orthophoto bands"), verbose_name=_("Orthophoto Bands")) + size = models.FloatField(default=0.0, blank=True, help_text=_("Size of the task on disk in megabytes"), verbose_name=_("Size")) class Meta: verbose_name = _("Task") @@ -1161,3 +1162,16 @@ def handle_images_upload(self, files): else: with open(file.temporary_file_path(), 'rb') as f: shutil.copyfileobj(f, fd) + + def update_size(self, commit=False): + try: + total_bytes = 0 + for dirpath, _, filenames in os.walk(self.task_path()): + for f in filenames: + fp = os.path.join(dirpath, f) + if not os.path.islink(fp): + total_bytes += os.path.getsize(fp) + self.size = (total_bytes / 1024 / 1024) + if commit: self.save() + except Exception as e: + logger.warn("Cannot update size for task {}: {}".format(self, str(e))) diff --git a/app/static/app/js/classes/Utils.js b/app/static/app/js/classes/Utils.js index 989da5309..e843275d0 100644 --- a/app/static/app/js/classes/Utils.js +++ b/app/static/app/js/classes/Utils.js @@ -93,6 +93,16 @@ export default { saveAs: function(text, filename){ var blob = new Blob([text], {type: "text/plain;charset=utf-8"}); FileSaver.saveAs(blob, filename); + }, + + // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript + bytesToSize: function(bytes, decimals = 2){ + if(bytes == 0) return '0 byte'; + var k = 1000; // or 1024 for binary + var dm = decimals || 3; + var sizes = ['bytes', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } }; diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index 256111380..c995e49fd 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -14,6 +14,7 @@ import PipelineSteps from '../classes/PipelineSteps'; import Css from '../classes/Css'; import Tags from '../classes/Tags'; import Trans from './Trans'; +import Utils from '../classes/Utils'; import { _, interpolate } from '../classes/gettext'; class TaskListItem extends React.Component { @@ -572,6 +573,11 @@ class TaskListItem extends React.Component { {_("Reconstructed Points:")} {stats.pointcloud.points.toLocaleString()} } + {task.size > 0 && + + {_("Size:")} + {Utils.bytesToSize(task.size * 1024 * 1024)} + } {_("Task Output:")}
diff --git a/app/static/app/js/components/UploadProgressBar.jsx b/app/static/app/js/components/UploadProgressBar.jsx index 689b22a10..e0bbe5d79 100644 --- a/app/static/app/js/components/UploadProgressBar.jsx +++ b/app/static/app/js/components/UploadProgressBar.jsx @@ -2,6 +2,7 @@ import '../css/UploadProgressBar.scss'; import React from 'react'; import PropTypes from 'prop-types'; import { _, interpolate } from '../classes/gettext'; +import Utils from '../classes/Utils'; class UploadProgressBar extends React.Component { static propTypes = { @@ -11,22 +12,12 @@ class UploadProgressBar extends React.Component { totalCount: PropTypes.number // number of files } - // http://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript - bytesToSize(bytes, decimals = 2){ - if(bytes == 0) return '0 byte'; - var k = 1000; // or 1024 for binary - var dm = decimals || 3; - var sizes = ['bytes', 'Kb', 'Mb', 'Gb', 'Tb', 'Pb', 'Eb', 'Zb', 'Yb']; - var i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; - } - render() { let percentage = (this.props.progress !== undefined ? this.props.progress : 0).toFixed(2); let bytes = this.props.totalBytesSent !== undefined && this.props.totalBytes !== undefined ? - ' ' + interpolate(_("remaining to upload: %(bytes)s"), { bytes: this.bytesToSize(this.props.totalBytes - this.props.totalBytesSent)}) : + ' ' + interpolate(_("remaining to upload: %(bytes)s"), { bytes: Utils.bytesToSize(this.props.totalBytes - this.props.totalBytesSent)}) : ""; let active = percentage < 100 ? "active" : ""; diff --git a/package.json b/package.json index 05af3481b..3887114cc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.0.3", + "version": "2.1.0", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { diff --git a/webodm/settings.py b/webodm/settings.py index aff1e75de..943a4705e 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -169,6 +169,7 @@ AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', # this is default 'guardian.backends.ObjectPermissionBackend', + 'app.auth.backends.ExternalBackend', ) # Internationalization @@ -380,6 +381,10 @@ def scalebyiv(color, n): # before it should be considered offline NODE_OFFLINE_MINUTES = 5 +USE_EXTERNAL_AUTH = True # TODO: change +EXTERNAL_AUTH_ENDPOINT = "http://192.168.2.253:5000/r/auth/login" +# TODO: make these env vars? + if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True From 08608a672729e45b844a41db1c888d7fbc9f193e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 21 Aug 2023 12:55:17 -0400 Subject: [PATCH 10/95] Fix non-georeferenced textured models loading --- app/static/app/js/ModelView.jsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/static/app/js/ModelView.jsx b/app/static/app/js/ModelView.jsx index f72b429d7..b149743e2 100644 --- a/app/static/app/js/ModelView.jsx +++ b/app/static/app/js/ModelView.jsx @@ -644,9 +644,10 @@ class ModelView extends React.Component { return; } - const offset = { - x: gltf.scene.CESIUM_RTC.center[0], - y: gltf.scene.CESIUM_RTC.center[1] + const offset = {x: 0, y: 0}; + if (gltf.scene.CESIUM_RTC && gltf.scene.CESIUM_RTC.center){ + offset.x = gltf.scene.CESIUM_RTC.center[0]; + offset.y = gltf.scene.CESIUM_RTC.center[1]; } addObject(gltf.scene, offset); From 5ba0d472afb7122c52c8668923b4a57127ff107c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 24 Aug 2023 12:17:50 -0400 Subject: [PATCH 11/95] Add tests, update size --- app/models/task.py | 2 ++ app/tests/test_api_task.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/app/models/task.py b/app/models/task.py index 09d51aa8a..79fb1da5b 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -433,6 +433,7 @@ def duplicate(self, set_new_name=True): shutil.copytree(self.task_path(), task.task_path()) else: logger.warning("Task {} doesn't have folder, will skip copying".format(self)) + return task except Exception as e: logger.warning("Cannot duplicate task: {}".format(str(e))) @@ -886,6 +887,7 @@ def extract_assets_and_complete(self): self.update_available_assets_field() self.update_epsg_field() self.update_orthophoto_bands_field() + self.update_size() self.potree_scene = {} self.running_progress = 1.0 self.console_output += gettext("Done!") + "\n" diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 343a53824..2cd2dbcb4 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -249,6 +249,9 @@ def test_task(self): # Orthophoto bands field should be an empty list self.assertEqual(len(task.orthophoto_bands), 0) + # Size should be zero + self.assertEqual(task.size, 0) + # tiles.json, bounds, metadata should not be accessible at this point tile_types = ['orthophoto', 'dsm', 'dtm'] endpoints = ['tiles.json', 'bounds', 'metadata'] @@ -384,6 +387,9 @@ def test_task(self): # Orthophoto bands field should be populated self.assertEqual(len(task.orthophoto_bands), 4) + # Size should be updated + self.assertTrue(task.size > 0) + # Can export orthophoto (when formula and bands are specified) res = client.post("/api/projects/{}/tasks/{}/orthophoto/export".format(project.id, task.id), { 'formula': 'NDVI' @@ -946,6 +952,7 @@ def connTimeout(*args, **kwargs): self.assertTrue(res.data['success']) new_task_id = res.data['task']['id'] self.assertNotEqual(res.data['task']['id'], task.id) + self.assertEqual(res.data['task']['size'], task.size) new_task = Task.objects.get(pk=new_task_id) From ba1965add0fc9645376b372d95f17577eead485c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 24 Aug 2023 15:02:30 -0400 Subject: [PATCH 12/95] Add profile model --- app/admin.py | 14 ++++++++++ app/migrations/0037_profile.py | 35 +++++++++++++++++++++++++ app/models/__init__.py | 1 + app/models/profile.py | 37 +++++++++++++++++++++++++++ app/templates/app/logged_in_base.html | 11 ++++++++ app/templatetags/settings.py | 9 +++++++ requirements.txt | 1 + webodm/settings.py | 10 ++++++++ 8 files changed, 118 insertions(+) create mode 100644 app/migrations/0037_profile.py create mode 100644 app/models/profile.py diff --git a/app/admin.py b/app/admin.py index 81848e199..7ad77d957 100644 --- a/app/admin.py +++ b/app/admin.py @@ -10,10 +10,13 @@ from django.urls import reverse from django.utils.html import format_html from guardian.admin import GuardedModelAdmin +from django.contrib.auth.admin import UserAdmin as BaseUserAdmin +from django.contrib.auth.models import User from app.models import PluginDatum from app.models import Preset from app.models import Plugin +from app.models import Profile from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin, valid_plugin, \ get_plugins_persistent_path, clear_plugins_cache, init_plugins from .models import Project, Task, Setting, Theme @@ -260,3 +263,14 @@ def plugin_actions(self, obj): admin.site.register(Plugin, PluginAdmin) + +class ProfileInline(admin.StackedInline): + model = Profile + can_delete = False + +class UserAdmin(BaseUserAdmin): + inlines = [ProfileInline] + +# Re-register UserAdmin +admin.site.unregister(User) +admin.site.register(User, UserAdmin) diff --git a/app/migrations/0037_profile.py b/app/migrations/0037_profile.py new file mode 100644 index 000000000..ab7a1fa08 --- /dev/null +++ b/app/migrations/0037_profile.py @@ -0,0 +1,35 @@ +# Generated by Django 2.2.27 on 2023-08-24 16:35 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +def create_profiles(apps, schema_editor): + User = apps.get_model('auth', 'User') + Profile = apps.get_model('app', 'Profile') + + for u in User.objects.all(): + p = Profile.objects.create(user=u) + p.save() + print("Created user profile for %s" % u.username) + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('app', '0036_task_size'), + ] + + operations = [ + migrations.CreateModel( + name='Profile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('quota', models.FloatField(blank=True, default=-1, help_text='Maximum disk quota in megabytes', verbose_name='Quota')), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + + migrations.RunPython(create_profiles), + ] diff --git a/app/models/__init__.py b/app/models/__init__.py index b7434b5d9..a9d64a244 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -5,6 +5,7 @@ from .setting import Setting from .plugin_datum import PluginDatum from .plugin import Plugin +from .profile import Profile # deprecated def image_directory_path(image_upload, filename): diff --git a/app/models/profile.py b/app/models/profile.py new file mode 100644 index 000000000..11adefee1 --- /dev/null +++ b/app/models/profile.py @@ -0,0 +1,37 @@ +from django.contrib.auth.models import User +from django.db import models +from django.utils.translation import gettext_lazy as _ +from django.db.models.signals import post_save +from django.dispatch import receiver +from app.models import Task +from django.db.models import Sum +from django.core.cache import cache + +class Profile(models.Model): + user = models.OneToOneField(User, on_delete=models.CASCADE) + quota = models.FloatField(default=-1, blank=True, help_text=_("Maximum disk quota in megabytes"), verbose_name=_("Quota")) + + def has_quota(self): + return self.quota != -1 + + def used_quota(self): + return Task.objects.filter(project__owner=self.user).aggregate(total=Sum('size'))['total'] + + def used_quota_cached(self): + k = f'used_quota_{self.user.id}' + cached = cache.get(k) + if cached is not None: + return cached + + v = self.used_quota() + cache.set(k, v, 300) # 2 minutes + return v + +@receiver(post_save, sender=User) +def create_user_profile(sender, instance, created, **kwargs): + if created: + Profile.objects.create(user=instance) + +@receiver(post_save, sender=User) +def save_user_profile(sender, instance, **kwargs): + instance.profile.save() \ No newline at end of file diff --git a/app/templates/app/logged_in_base.html b/app/templates/app/logged_in_base.html index c4d18f5c6..2f8fe229d 100644 --- a/app/templates/app/logged_in_base.html +++ b/app/templates/app/logged_in_base.html @@ -15,6 +15,17 @@ {% blocktrans with user=user.username %}Hello, {{ user }}!{% endblocktrans %}
+ {% if user.profile.has_quota %} +
  • + {% with tot_quota=user.profile.quota %} + {% with used_quota=user.profile.used_quota_cached %} + {% percentage 0 tot_quota as perc_quota %} + + Tot: {{ tot_quota }} Used: {{ used_quota }} + Perc: {{ perc_quota|floatformat:0 }} + + {% endwith %}{% endwith %} + {% endif %}
  • {% trans 'Logout' %}
    • diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index 7bda2e18a..f12ebf882 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -7,6 +7,15 @@ register = template.Library() logger = logging.getLogger('app.logger') +@register.simple_tag +def percentage(num, den, maximum=None): + if den == 0: + return 0 + perc = max(0, num / den * 100) + if maximum is not None: + perc = min(perc, maximum) + return perc + @register.simple_tag def is_single_user_mode(): return settings.SINGLE_USER_MODE diff --git a/requirements.txt b/requirements.txt index 61aa2130b..07a92e66d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ django-filter==2.4.0 django-guardian==1.4.9 django-imagekit==4.0.1 django-libsass==0.7 +django-redis==4.12.1 django-webpack-loader==0.6.0 djangorestframework==3.13.1 djangorestframework-jwt==1.9.0 diff --git a/webodm/settings.py b/webodm/settings.py index 943a4705e..66bf8561e 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -377,6 +377,16 @@ def scalebyiv(color, n): CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": os.environ.get('WO_BROKER', 'redis://localhost'), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + } + } +} + # Number of minutes a processing node hasn't been seen # before it should be considered offline NODE_OFFLINE_MINUTES = 5 From cd7f7790198c374701f8f53365707a4a0b7ad4cb Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 26 Aug 2023 06:07:05 -0400 Subject: [PATCH 13/95] Update locale --- locale | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/locale b/locale index 31a7b8fc6..04c6bb88e 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 31a7b8fc6d955e8bd6c13d2de84501bc43895190 +Subproject commit 04c6bb88e48e5dad3c0686dafa59ae8852d72273 From a0dbd681221b6ad5d0262b0bafbf7e7795bfef68 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 26 Aug 2023 08:12:16 -0400 Subject: [PATCH 14/95] Pretty quota status bar --- app/static/app/css/sb-admin-2.css | 17 +++++++++++++++- app/templates/app/logged_in_base.html | 28 ++++++++++++++++++++------- app/templatetags/settings.py | 14 +++++++++++++- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/app/static/app/css/sb-admin-2.css b/app/static/app/css/sb-admin-2.css index 396503dbb..8103ba6be 100644 --- a/app/static/app/css/sb-admin-2.css +++ b/app/static/app/css/sb-admin-2.css @@ -50,11 +50,26 @@ body { margin-right: 0; } -.navbar-top-links .dropdown-menu li a { +.navbar-top-links .dropdown-menu li a{ padding: 3px 20px; min-height: 0; } +.navbar-top-links .dropdown-menu li div.info-item{ + padding: 3px 8px; + min-height: 0; +} + +.navbar-top-links .dropdown-menu li div.info-item.quotas{ + min-width: 190px; +} + +.navbar-top-links .dropdown-menu li .progress{ + margin-bottom: 0; + margin-top: 6px; +} + + .navbar-top-links .dropdown-menu li a div { white-space: normal; } diff --git a/app/templates/app/logged_in_base.html b/app/templates/app/logged_in_base.html index 2f8fe229d..2920cdfb8 100644 --- a/app/templates/app/logged_in_base.html +++ b/app/templates/app/logged_in_base.html @@ -12,17 +12,31 @@
      {% if user.profile.has_quota %} -
    • +
    • + {% with tot_quota=user.profile.quota %} {% with used_quota=user.profile.used_quota_cached %} - {% percentage 0 tot_quota as perc_quota %} - - Tot: {{ tot_quota }} Used: {{ used_quota }} - Perc: {{ perc_quota|floatformat:0 }} + {% percentage used_quota tot_quota as perc_quota %} + {% percentage used_quota tot_quota 100 as bar_width %} + +
    • +
      + {% with usage=perc_quota|floatformat:0 used=used_quota|storage_size total=tot_quota|storage_size %} + {% blocktrans %}{{used}} of {{total}} used{% endblocktrans %} +
      +
      + {{usage}}% +
      +
      + {% endwith %} +
      +
      • {% endwith %}{% endwith %} {% endif %} diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index f12ebf882..ae6e2c7cd 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -1,5 +1,5 @@ import datetime - +import math import logging from django import template from webodm import settings @@ -7,6 +7,18 @@ register = template.Library() logger = logging.getLogger('app.logger') +@register.filter +def storage_size(megabytes): + k = 1000 + k2 = k ** 2 + k3 = k ** 3 + if megabytes <= k2: + return str(round(megabytes / k, 2)) + ' GB' + elif megabytes <= k3: + return str(round(megabytes / k2, 2)) + ' TB' + else: + return str(round(megabytes / k3, 2)) + ' PB' + @register.simple_tag def percentage(num, den, maximum=None): if den == 0: From 872d5abbc774f6c1d346274c516f9536934d938f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 26 Aug 2023 09:23:42 -0400 Subject: [PATCH 15/95] Invalidate cache, warn of quota excess --- app/models/profile.py | 12 +++++++++++- app/models/task.py | 5 +++++ app/templates/app/dashboard.html | 10 ++++++++++ app/templates/app/logged_in_base.html | 5 ++--- app/templatetags/settings.py | 4 ++++ app/views/app.py | 5 +++-- webodm/settings.py | 5 +++++ 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/app/models/profile.py b/app/models/profile.py index 11adefee1..d77d59323 100644 --- a/app/models/profile.py +++ b/app/models/profile.py @@ -24,8 +24,18 @@ def used_quota_cached(self): return cached v = self.used_quota() - cache.set(k, v, 300) # 2 minutes + cache.set(k, v, 1800) # 30 minutes return v + + def has_exceeded_quota_cached(self): + if not self.has_quota(): + return False + + q = self.used_quota_cached() + return q > self.quota + + def clear_used_quota_cache(self): + cache.delete(f'used_quota_{self.user.id}') @receiver(post_save, sender=User) def create_user_profile(sender, instance, created, **kwargs): diff --git a/app/models/task.py b/app/models/task.py index 79fb1da5b..4235011cf 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -434,6 +434,7 @@ def duplicate(self, set_new_name=True): else: logger.warning("Task {} doesn't have folder, will skip copying".format(self)) + self.project.owner.profile.clear_used_quota_cache() return task except Exception as e: logger.warning("Cannot duplicate task: {}".format(str(e))) @@ -1037,6 +1038,8 @@ def delete(self, using=None, keep_parents=False): except FileNotFoundError as e: logger.warning(e) + self.project.owner.profile.clear_used_quota_cache() + plugin_signals.task_removed.send_robust(sender=self.__class__, task_id=task_id) def set_failure(self, error_message): @@ -1175,5 +1178,7 @@ def update_size(self, commit=False): total_bytes += os.path.getsize(fp) self.size = (total_bytes / 1024 / 1024) if commit: self.save() + + self.project.owner.profile.clear_used_quota_cache() except Exception as e: logger.warn("Cannot update size for task {}: {}".format(self, str(e))) diff --git a/app/templates/app/dashboard.html b/app/templates/app/dashboard.html index 1abb2f921..b6b280159 100644 --- a/app/templates/app/dashboard.html +++ b/app/templates/app/dashboard.html @@ -1,5 +1,6 @@ {% extends "app/logged_in_base.html" %} {% load i18n %} +{% load settings %} {% block content %} {% load render_bundle from webpack_loader %} @@ -39,6 +40,15 @@

      {% trans 'Welcome!' %} ☺

      {% endif %} + {% if user.profile.has_exceeded_quota_cached %} + {% with total=user.profile.quota|storage_size used=user.profile.used_quota_cached|storage_size %} + {% quota_exceeded_grace_period as hours %} +
      + {% blocktrans %}The current storage quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted within {{ hours }} hours, until usage falls below {{ total }}.{% endblocktrans %} +
      + {% endwith %} + {% endif %} +
      {% endif %} diff --git a/app/templates/app/logged_in_base.html b/app/templates/app/logged_in_base.html index 2920cdfb8..3abe707c0 100644 --- a/app/templates/app/logged_in_base.html +++ b/app/templates/app/logged_in_base.html @@ -20,8 +20,7 @@ {% if user.profile.has_quota %}
    • - {% with tot_quota=user.profile.quota %} - {% with used_quota=user.profile.used_quota_cached %} + {% with tot_quota=user.profile.quota used_quota=user.profile.used_quota_cached %} {% percentage used_quota tot_quota as perc_quota %} {% percentage used_quota tot_quota 100 as bar_width %} @@ -38,7 +37,7 @@
    - {% endwith %}{% endwith %} + {% endwith %} {% endif %}
  • {% trans 'Logout' %} diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index ae6e2c7cd..3ab491daf 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -28,6 +28,10 @@ def percentage(num, den, maximum=None): perc = min(perc, maximum) return perc +@register.simple_tag +def quota_exceeded_grace_period(): + return settings.QUOTA_EXCEEDED_GRACE_PERIOD + @register.simple_tag def is_single_user_mode(): return settings.SINGLE_USER_MODE diff --git a/app/views/app.py b/app/views/app.py index 58dbc9079..f37266e24 100644 --- a/app/views/app.py +++ b/app/views/app.py @@ -38,9 +38,10 @@ def dashboard(request): return redirect(settings.PROCESSING_NODES_ONBOARDING) no_tasks = Task.objects.filter(project__owner=request.user).count() == 0 - + no_projects = Project.objects.filter(owner=request.user).count() == 0 + # Create first project automatically - if Project.objects.count() == 0: + if no_projects and request.user.has_perm('app.add_project'): Project.objects.create(owner=request.user, name=_("First Project")) return render(request, 'app/dashboard.html', {'title': _('Dashboard'), diff --git a/webodm/settings.py b/webodm/settings.py index 66bf8561e..896b27e97 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -395,6 +395,11 @@ def scalebyiv(color, n): EXTERNAL_AUTH_ENDPOINT = "http://192.168.2.253:5000/r/auth/login" # TODO: make these env vars? +# Number of hours before tasks are automatically deleted +# from an account that is exceeding a disk quota +QUOTA_EXCEEDED_GRACE_PERIOD = 8 + + if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True From b4e54e6406b1b87a01831cf753a4287a71f68252 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 26 Aug 2023 09:53:42 -0400 Subject: [PATCH 16/95] Tweaks --- app/static/app/js/components/TaskListItem.jsx | 2 +- app/templates/app/dashboard.html | 4 ++-- app/templates/app/logged_in_base.html | 2 +- app/templatetags/settings.py | 2 +- app/tests/test_api_task.py | 3 +++ 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index c995e49fd..a8f581147 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -575,7 +575,7 @@ class TaskListItem extends React.Component { } {task.size > 0 && - {_("Size:")} + {_("Disk Usage:")} {Utils.bytesToSize(task.size * 1024 * 1024)} } diff --git a/app/templates/app/dashboard.html b/app/templates/app/dashboard.html index b6b280159..8fa7a5486 100644 --- a/app/templates/app/dashboard.html +++ b/app/templates/app/dashboard.html @@ -41,10 +41,10 @@

    {% trans 'Welcome!' %} ☺

    {% endif %} {% if user.profile.has_exceeded_quota_cached %} - {% with total=user.profile.quota|storage_size used=user.profile.used_quota_cached|storage_size %} + {% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %} {% quota_exceeded_grace_period as hours %}
    - {% blocktrans %}The current storage quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted within {{ hours }} hours, until usage falls below {{ total }}.{% endblocktrans %} + {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted within {{ hours }} hours, until usage falls below {{ total }}.{% endblocktrans %}
    {% endwith %} {% endif %} diff --git a/app/templates/app/logged_in_base.html b/app/templates/app/logged_in_base.html index 3abe707c0..8ba8d36f6 100644 --- a/app/templates/app/logged_in_base.html +++ b/app/templates/app/logged_in_base.html @@ -26,7 +26,7 @@
  • - {% with usage=perc_quota|floatformat:0 used=used_quota|storage_size total=tot_quota|storage_size %} + {% with usage=perc_quota|floatformat:0 used=used_quota|disk_size total=tot_quota|disk_size %} {% blocktrans %}{{used}} of {{total}} used{% endblocktrans %}
    diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index 3ab491daf..2cb0d7235 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -8,7 +8,7 @@ logger = logging.getLogger('app.logger') @register.filter -def storage_size(megabytes): +def disk_size(megabytes): k = 1000 k2 = k ** 2 k3 = k ** 3 diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 2cd2dbcb4..1abbff4ec 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -390,6 +390,9 @@ def test_task(self): # Size should be updated self.assertTrue(task.size > 0) + # The owner's used quota should have increased + self.assertTrue(task.project.owner.profile.used_quota_cached() > 0) + # Can export orthophoto (when formula and bands are specified) res = client.post("/api/projects/{}/tasks/{}/orthophoto/export".format(project.id, task.id), { 'formula': 'NDVI' From aa737da1a1dd16948af143bb93dbef1aec23ed04 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 1 Sep 2023 16:16:13 -0400 Subject: [PATCH 17/95] Expose profiles API, quota update endpoint --- app/api/admin.py | 39 +++++++++++++++++++++++++++++++- app/api/urls.py | 5 ++-- app/models/profile.py | 26 ++++++++++++++++++++- app/templates/app/dashboard.html | 4 ++-- app/templatetags/settings.py | 21 ++++++++++++++--- worker/celery.py | 8 +++++++ worker/tasks.py | 16 ++++++++++++- 7 files changed, 109 insertions(+), 10 deletions(-) diff --git a/app/api/admin.py b/app/api/admin.py index 329e00ef8..2de55e4dd 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -1,7 +1,10 @@ from django.contrib.auth.models import User, Group -from rest_framework import serializers, viewsets, generics, status +from app.models import Profile +from rest_framework import serializers, viewsets, generics, status, exceptions +from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser from rest_framework.response import Response +from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth.hashers import make_password from app import models @@ -20,6 +23,7 @@ def get_queryset(self): if email is not None: queryset = queryset.filter(email=email) return queryset + def create(self, request): data = request.data.copy() password = data.get('password') @@ -44,3 +48,36 @@ def get_queryset(self): if name is not None: queryset = queryset.filter(name=name) return queryset + + +class ProfileSerializer(serializers.ModelSerializer): + class Meta: + model = Profile + exclude = ('id', ) + + read_only_fields = ('user', ) + +class AdminProfileViewSet(viewsets.ModelViewSet): + serializer_class = ProfileSerializer + permission_classes = [IsAdminUser] + lookup_field = 'user' + + def get_queryset(self): + return Profile.objects.all() + + + @action(detail=True, methods=['post']) + def update_quota_deadline(self, request, user=None): + try: + hours = float(request.data.get('hours', '')) + if hours < 0: + raise ValueError("hours must be >= 0") + except ValueError as e: + raise exceptions.ValidationError(str(e)) + + try: + p = Profile.objects.get(user=user) + except ObjectDoesNotExist: + raise exceptions.NotFound() + + return Response({'deadline': p.set_quota_deadline(hours)}, status=status.HTTP_200_OK) diff --git a/app/api/urls.py b/app/api/urls.py index bebfccd4f..a29f0e0ad 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -6,7 +6,7 @@ from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport from .imageuploads import Thumbnail, ImageDownload from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView -from .admin import AdminUserViewSet, AdminGroupViewSet +from .admin import AdminUserViewSet, AdminGroupViewSet, AdminProfileViewSet from rest_framework_nested import routers from rest_framework_jwt.views import obtain_jwt_token from .tiler import TileJson, Bounds, Metadata, Tiles, Export @@ -26,6 +26,7 @@ admin_router = routers.DefaultRouter() admin_router.register(r'admin/users', AdminUserViewSet, basename='admin-users') admin_router.register(r'admin/groups', AdminGroupViewSet, basename='admin-groups') +admin_router.register(r'admin/profiles', AdminProfileViewSet, basename='admin-groups') urlpatterns = [ url(r'processingnodes/options/$', ProcessingNodeOptionsView.as_view()), @@ -56,7 +57,7 @@ url(r'^auth/', include('rest_framework.urls')), url(r'^token-auth/', obtain_jwt_token), - url(r'^plugins/(?P[^/.]+)/(.*)$', api_view_handler) + url(r'^plugins/(?P[^/.]+)/(.*)$', api_view_handler), ] if settings.ENABLE_USERS_API: diff --git a/app/models/profile.py b/app/models/profile.py index d77d59323..1a1cf9719 100644 --- a/app/models/profile.py +++ b/app/models/profile.py @@ -1,3 +1,4 @@ +import time from django.contrib.auth.models import User from django.db import models from django.utils.translation import gettext_lazy as _ @@ -6,6 +7,8 @@ from app.models import Task from django.db.models import Sum from django.core.cache import cache +from webodm import settings + class Profile(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE) @@ -17,6 +20,13 @@ def has_quota(self): def used_quota(self): return Task.objects.filter(project__owner=self.user).aggregate(total=Sum('size'))['total'] + def has_exceeded_quota(self): + if not self.has_quota(): + return False + + q = self.used_quota() + return q > self.quota + def used_quota_cached(self): k = f'used_quota_{self.user.id}' cached = cache.get(k) @@ -36,6 +46,20 @@ def has_exceeded_quota_cached(self): def clear_used_quota_cache(self): cache.delete(f'used_quota_{self.user.id}') + + def get_quota_deadline(self): + return cache.get(f'quota_deadline_{self.user.id}') + + def set_quota_deadline(self, hours): + k = f'quota_deadline_{self.user.id}' + seconds = (hours * 60 * 60) + v = time.time() + seconds + cache.set(k, v, int(max(seconds * 10, settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60))) + return v + + def clear_quota_deadline(self): + cache.delete(f'quota_deadline_{self.user.id}') + @receiver(post_save, sender=User) def create_user_profile(sender, instance, created, **kwargs): @@ -44,4 +68,4 @@ def create_user_profile(sender, instance, created, **kwargs): @receiver(post_save, sender=User) def save_user_profile(sender, instance, **kwargs): - instance.profile.save() \ No newline at end of file + instance.profile.save() diff --git a/app/templates/app/dashboard.html b/app/templates/app/dashboard.html index 8fa7a5486..668af63e4 100644 --- a/app/templates/app/dashboard.html +++ b/app/templates/app/dashboard.html @@ -42,9 +42,9 @@

    {% trans 'Welcome!' %} ☺

    {% if user.profile.has_exceeded_quota_cached %} {% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %} - {% quota_exceeded_grace_period as hours %} + {% quota_exceeded_grace_period as when %}
    - {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted within {{ hours }} hours, until usage falls below {{ total }}.{% endblocktrans %} + {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted {{ when }}, until usage falls below {{ total }}.{% endblocktrans %}
    {% endwith %} {% endif %} diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index 2cb0d7235..8ed581fa4 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -1,8 +1,10 @@ import datetime import math import logging +import time from django import template from webodm import settings +from django.utils.translation import gettext as _ register = template.Library() logger = logging.getLogger('app.logger') @@ -28,9 +30,22 @@ def percentage(num, den, maximum=None): perc = min(perc, maximum) return perc -@register.simple_tag -def quota_exceeded_grace_period(): - return settings.QUOTA_EXCEEDED_GRACE_PERIOD +@register.simple_tag(takes_context=True) +def quota_exceeded_grace_period(context): + deadline = context.request.user.profile.get_quota_deadline() + now = time.time() + if deadline is None: + deadline = now + settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60 + diff = max(0, deadline - now) + if diff >= 60*60*24*2: + return _("within %(num)s days") % {"num": math.ceil(diff / (60*60*24))} + elif diff >= 60*60: + return _("within %(num)s hours") % {"num": math.ceil(diff / (60*60))} + elif diff > 0: + return _("within %(num)s minutes") % {"num": math.ceil(diff / 60)} + else: + return _("very soon") + @register.simple_tag def is_single_user_mode(): diff --git a/worker/celery.py b/worker/celery.py index 083edd814..cb9209e1c 100644 --- a/worker/celery.py +++ b/worker/celery.py @@ -44,6 +44,14 @@ 'retry': False } }, + 'check-quotas': { + 'task': 'worker.tasks.check_quotas', + 'schedule': 3600, + 'options': { + 'expires': 1799, + 'retry': False + } + }, } # Mock class for handling async results during testing diff --git a/worker/tasks.py b/worker/tasks.py index b220b3e36..7b8ef84e9 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -11,6 +11,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count from django.db.models import Q +from app.models import Profile from app.models import Project from app.models import Task @@ -202,4 +203,17 @@ def export_pointcloud(self, input, **opts): return result except Exception as e: logger.error(str(e)) - return {'error': str(e)} \ No newline at end of file + return {'error': str(e)} + +@app.task +def check_quotas(): + profiles = Profile.objects.filter(quota__gt=-1) + # for p in profiles: + # deadline_key = "%s_quota_exceeded_deadline" % p.user.id + + # if p.has_exceeded_quota(): + # now = time.time() + # deadline = redis_client.getset(deadline_key, now + (settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60)) + # # if deadline < now: TODO.. + # else: + # redis_client.delete(deadline_key) From f5ff31b3ff2d870a427ebe7c9a8a730a149b23b6 Mon Sep 17 00:00:00 2001 From: Grant Date: Sat, 2 Sep 2023 17:29:44 +0100 Subject: [PATCH 18/95] Update Map.jsx to use correct tile.osm.org URL See: https://github.com/openstreetmap/operations/issues/737 --- app/static/app/js/components/Map.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index 7cf2a2480..810b66661 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -399,14 +399,14 @@ class Map extends React.Component { const customLayer = L.layerGroup(); customLayer.on("add", a => { - const defaultCustomBm = window.localStorage.getItem('lastCustomBasemap') || 'https://a.tile.openstreetmap.org/{z}/{x}/{y}.png'; + const defaultCustomBm = window.localStorage.getItem('lastCustomBasemap') || 'https://tile.openstreetmap.org/{z}/{x}/{y}.png'; let url = window.prompt([_('Enter a tile URL template. Valid coordinates are:'), _('{z}, {x}, {y} for Z/X/Y tile scheme'), _('{-y} for flipped TMS-style Y coordinates'), '', _('Example:'), -'https://a.tile.openstreetmap.org/{z}/{x}/{y}.png'].join("\n"), defaultCustomBm); +'https://tile.openstreetmap.org/{z}/{x}/{y}.png'].join("\n"), defaultCustomBm); if (url){ customLayer.clearLayers(); From f1b358db44b7db2e87e90d6c9f0f2f7ef4ea6b92 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 4 Sep 2023 11:06:27 -0400 Subject: [PATCH 19/95] Pin redis version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 04daa1c45..23b8922fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: restart: unless-stopped oom_score_adj: 0 broker: - image: redis + image: redis:7.0.10 container_name: broker restart: unless-stopped oom_score_adj: -500 From 1b92ee1f19ea2f9e69cbddca46f92fd6ee0b8302 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 4 Sep 2023 13:34:54 -0400 Subject: [PATCH 20/95] Quota deletion working --- app/static/app/css/sb-admin-2.css | 2 +- app/templatetags/settings.py | 2 +- worker/tasks.py | 32 ++++++++++++++++++++++--------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/app/static/app/css/sb-admin-2.css b/app/static/app/css/sb-admin-2.css index 8103ba6be..2d19dd0c4 100644 --- a/app/static/app/css/sb-admin-2.css +++ b/app/static/app/css/sb-admin-2.css @@ -61,7 +61,7 @@ body { } .navbar-top-links .dropdown-menu li div.info-item.quotas{ - min-width: 190px; + min-width: 232px; } .navbar-top-links .dropdown-menu li .progress{ diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index 8ed581fa4..a540ae5eb 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -41,7 +41,7 @@ def quota_exceeded_grace_period(context): return _("within %(num)s days") % {"num": math.ceil(diff / (60*60*24))} elif diff >= 60*60: return _("within %(num)s hours") % {"num": math.ceil(diff / (60*60))} - elif diff > 0: + elif diff > 1: return _("within %(num)s minutes") % {"num": math.ceil(diff / 60)} else: return _("very soon") diff --git a/worker/tasks.py b/worker/tasks.py index 7b8ef84e9..3cafcdbd4 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -208,12 +208,26 @@ def export_pointcloud(self, input, **opts): @app.task def check_quotas(): profiles = Profile.objects.filter(quota__gt=-1) - # for p in profiles: - # deadline_key = "%s_quota_exceeded_deadline" % p.user.id - - # if p.has_exceeded_quota(): - # now = time.time() - # deadline = redis_client.getset(deadline_key, now + (settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60)) - # # if deadline < now: TODO.. - # else: - # redis_client.delete(deadline_key) + for p in profiles: + if p.has_exceeded_quota(): + deadline = p.get_quota_deadline() + if deadline is None: + deadline = p.set_quota_deadline(settings.QUOTA_EXCEEDED_GRACE_PERIOD) + now = time.time() + if now > deadline: + # deadline passed, delete tasks until quota is met + logger.info("Quota deadline expired for %s, deleting tasks" % str(p.user.username)) + + while p.has_exceeded_quota(): + try: + last_task = Task.objects.filter(project__owner=p.user).order_by("-created_at").first() + if last_task is None: + break + logger.info("Deleting %s" % last_task) + last_task.delete() + except Exception as e: + logger.warn("Cannot delete %s for %s: %s" % (str(last_task), str(p.user.username), str(e))) + break + else: + p.clear_quota_deadline() + From 5047413e124a234a801b4d31ba0df785cb839902 Mon Sep 17 00:00:00 2001 From: Chris Bateman Date: Tue, 5 Sep 2023 09:01:51 +1000 Subject: [PATCH 21/95] Update test-docker.yml Update actions/checkout from v2 to v3 --- .github/workflows/test-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 7780d000a..9cc0ce979 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -7,7 +7,7 @@ jobs: docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: 'recursive' name: Checkout From c4c5085e2af3b10cb2c30faea886890e5ab64766 Mon Sep 17 00:00:00 2001 From: Chris Bateman Date: Tue, 5 Sep 2023 09:15:58 +1000 Subject: [PATCH 22/95] Update build-docs.yml Update actions/checkout from v2 to v3 --- .github/workflows/build-docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 7ebcf9963..3926d2e6e 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -12,7 +12,7 @@ jobs: ruby-version: 2.7 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: @@ -28,4 +28,4 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./slate/build - keep_files: true \ No newline at end of file + keep_files: true From 132e8f9d699cf3eb4131bb7545a62a38b8f1fb56 Mon Sep 17 00:00:00 2001 From: Chris Bateman Date: Tue, 5 Sep 2023 09:17:01 +1000 Subject: [PATCH 23/95] Update build-and-publish.yml Update actions/checkout from v2 to v3 --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index a274ad249..bc756fd34 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: 'recursive' - name: Set up QEMU From 4cd5a01023c1c1d4b80e1adeeeaa1d630ae12bbd Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 6 Sep 2023 11:09:49 -0400 Subject: [PATCH 24/95] Add --external-auth-endpoint --- .env | 1 + app/auth/backends.py | 21 ++++++++++++++++----- app/models/profile.py | 5 ++++- docker-compose.yml | 2 ++ webodm.sh | 8 ++++++++ webodm/settings.py | 4 +--- 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.env b/.env index 5ff6f7a09..516bcd214 100644 --- a/.env +++ b/.env @@ -10,3 +10,4 @@ WO_DEBUG=NO WO_DEV=NO WO_BROKER=redis://broker WO_DEFAULT_NODES=1 +WO_EXTERNAL_AUTH_ENDPOINT= diff --git a/app/auth/backends.py b/app/auth/backends.py index 95fcb2c12..c25f2be3d 100644 --- a/app/auth/backends.py +++ b/app/auth/backends.py @@ -2,7 +2,7 @@ from django.contrib.auth.backends import ModelBackend from django.contrib.auth.models import User from nodeodm.models import ProcessingNode -from webodm.settings import EXTERNAL_AUTH_ENDPOINT, USE_EXTERNAL_AUTH +from webodm.settings import EXTERNAL_AUTH_ENDPOINT from guardian.shortcuts import assign_perm import logging @@ -10,7 +10,7 @@ class ExternalBackend(ModelBackend): def authenticate(self, request, username=None, password=None): - if not USE_EXTERNAL_AUTH: + if EXTERNAL_AUTH_ENDPOINT == "": return None try: @@ -20,10 +20,10 @@ def authenticate(self, request, username=None, password=None): }, headers={'Accept': 'application/json'}) res = r.json() + # logger.info(res) + if 'message' in res or 'error' in res: return None - - logger.info(res) if 'user_id' in res: try: @@ -33,6 +33,17 @@ def authenticate(self, request, username=None, password=None): if user.username != username: user.username = username user.save() + + # Update quotas + maxQuota = -1 + if 'maxQuota' in res: + maxQuota = res['maxQuota'] + if 'node' in res and 'limits' in res['node'] and 'maxQuota' in res['node']['limits']: + maxQuota = res['node']['limits']['maxQuota'] + + if user.profile.quota != maxQuota: + user.profile.quota = maxQuota + user.save() except User.DoesNotExist: user = User(pk=res['user_id'], username=username) user.save() @@ -64,7 +75,7 @@ def authenticate(self, request, username=None, password=None): return None def get_user(self, user_id): - if not USE_EXTERNAL_AUTH: + if EXTERNAL_AUTH_ENDPOINT == "": return None try: diff --git a/app/models/profile.py b/app/models/profile.py index 1a1cf9719..54e227d1d 100644 --- a/app/models/profile.py +++ b/app/models/profile.py @@ -18,7 +18,10 @@ def has_quota(self): return self.quota != -1 def used_quota(self): - return Task.objects.filter(project__owner=self.user).aggregate(total=Sum('size'))['total'] + q = Task.objects.filter(project__owner=self.user).aggregate(total=Sum('size'))['total'] + if q is None: + q = 0 + return q def has_exceeded_quota(self): if not self.has_quota(): diff --git a/docker-compose.yml b/docker-compose.yml index 04daa1c45..2b79fa64d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: - WO_BROKER - WO_DEV - WO_DEV_WATCH_PLUGINS + - WO_EXTERNAL_AUTH_ENDPOINT restart: unless-stopped oom_score_adj: 0 broker: @@ -52,5 +53,6 @@ services: environment: - WO_BROKER - WO_DEBUG + - WO_EXTERNAL_AUTH_ENDPOINT restart: unless-stopped oom_score_adj: 250 diff --git a/webodm.sh b/webodm.sh index b979a09d8..4a044ba00 100755 --- a/webodm.sh +++ b/webodm.sh @@ -130,6 +130,12 @@ case $key in shift # past argument shift # past value ;; + --external-auth-endpoint) + WO_EXTERNAL_AUTH_ENDPOINT="$2" + export WO_EXTERNAL_AUTH_ENDPOINT + shift # past argument + shift # past value + ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument @@ -170,6 +176,7 @@ usage(){ echo " --broker Set the URL used to connect to the celery broker (default: $DEFAULT_BROKER)" echo " --detached Run WebODM in detached mode. This means WebODM will run in the background, without blocking the terminal (default: disabled)" echo " --gpu Use GPU NodeODM nodes (Linux only) (default: disabled)" + echo " --external-auth-endpoint External authentication endpoint (default: disabled)" exit } @@ -339,6 +346,7 @@ start(){ echo "SSL insecure port redirect: $WO_SSL_INSECURE_PORT_REDIRECT" echo "Celery Broker: $WO_BROKER" echo "Default Nodes: $WO_DEFAULT_NODES" + echo "External auth endpoint: $WO_EXTERNAL_AUTH_ENDPOINT" echo "================================" echo "Make sure to issue a $0 down if you decide to change the environment." echo "" diff --git a/webodm/settings.py b/webodm/settings.py index 896b27e97..9774338f8 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -391,9 +391,7 @@ def scalebyiv(color, n): # before it should be considered offline NODE_OFFLINE_MINUTES = 5 -USE_EXTERNAL_AUTH = True # TODO: change -EXTERNAL_AUTH_ENDPOINT = "http://192.168.2.253:5000/r/auth/login" -# TODO: make these env vars? +EXTERNAL_AUTH_ENDPOINT = os.environ.get('WO_EXTERNAL_AUTH_ENDPOINT', '') # Number of hours before tasks are automatically deleted # from an account that is exceeding a disk quota From bd70b4b7ec28c1f16c161e9c99a65b56b8465318 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 7 Sep 2023 10:50:44 -0400 Subject: [PATCH 25/95] Increase point budget, auto-login logic PoC --- app/static/app/js/ModelView.jsx | 2 +- app/templates/app/registration/login.html | 37 ++++++++++++++++++++++- app/templatetags/settings.py | 4 +++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/app/static/app/js/ModelView.jsx b/app/static/app/js/ModelView.jsx index b149743e2..3a0478ce1 100644 --- a/app/static/app/js/ModelView.jsx +++ b/app/static/app/js/ModelView.jsx @@ -298,7 +298,7 @@ class ModelView extends React.Component { window.viewer = new Potree.Viewer(container); viewer.setEDLEnabled(true); viewer.setFOV(60); - viewer.setPointBudget(1*1000*1000); + viewer.setPointBudget(10*1000*1000); viewer.setEDLEnabled(true); viewer.loadSettingsFromURL(); diff --git a/app/templates/app/registration/login.html b/app/templates/app/registration/login.html index 164a46e3e..9945dd422 100644 --- a/app/templates/app/registration/login.html +++ b/app/templates/app/registration/login.html @@ -10,11 +10,12 @@ {% endif %} {% is_single_user_mode as autologin %} + {% external_auth_endpoint as ext_auth_ep %} {% if autologin %} {% else %} -
    {% csrf_token %} + {% csrf_token %} {% for field in form %} {% include 'registration/form_field.html' %} {% endfor %} @@ -34,5 +35,39 @@
    + + {% if ext_auth_ep != '' %} +
    + +
    + + {% endif %} + {% endif %} {% endblock %} \ No newline at end of file diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index a540ae5eb..e439e161c 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -9,6 +9,10 @@ register = template.Library() logger = logging.getLogger('app.logger') +@register.simple_tag +def external_auth_endpoint(): + return settings.EXTERNAL_AUTH_ENDPOINT + @register.filter def disk_size(megabytes): k = 1000 From 73052fb2ec60d6b9dd535107a4509478d5425bac Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 8 Sep 2023 12:28:13 -0400 Subject: [PATCH 26/95] External auto auth working --- app/api/externalauth.py | 38 ++++++++ app/api/urls.py | 4 + app/auth/backends.py | 103 +++++++++++----------- app/templates/app/registration/login.html | 43 +++++---- app/templatetags/settings.py | 4 +- 5 files changed, 124 insertions(+), 68 deletions(-) create mode 100644 app/api/externalauth.py diff --git a/app/api/externalauth.py b/app/api/externalauth.py new file mode 100644 index 000000000..dcc96a609 --- /dev/null +++ b/app/api/externalauth.py @@ -0,0 +1,38 @@ +from django.contrib.auth.models import User +from django.contrib.auth import login +from rest_framework.views import APIView +from rest_framework import exceptions, permissions, parsers +from rest_framework.response import Response +from app.auth.backends import get_user_from_external_auth_response +import requests +from webodm import settings + +class ExternalTokenAuth(APIView): + permission_classes = (permissions.AllowAny,) + parser_classes = (parsers.JSONParser, parsers.FormParser,) + + def post(self, request): + # This should never happen + if settings.EXTERNAL_AUTH_ENDPOINT == '': + return Response({'error': 'EXTERNAL_AUTH_ENDPOINT not set'}) + + token = request.COOKIES.get('external_access_token', '') + if token == '': + return Response({'error': 'external_access_token cookie not set'}) + + try: + r = requests.post(settings.EXTERNAL_AUTH_ENDPOINT, headers={ + 'Authorization': "Bearer %s" % token + }) + res = r.json() + if res.get('user_id') is not None: + user = get_user_from_external_auth_response(res) + if user is not None: + login(request, user, backend='django.contrib.auth.backends.ModelBackend') + return Response({'redirect': '/'}) + else: + return Response({'error': 'Invalid credentials'}) + else: + return Response({'error': res.get('message', 'Invalid external server response')}) + except Exception as e: + return Response({'error': str(e)}) diff --git a/app/api/urls.py b/app/api/urls.py index a29f0e0ad..3de135901 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -13,6 +13,7 @@ from .potree import Scene, CameraView from .workers import CheckTask, GetTaskResult from .users import UsersList +from .externalauth import ExternalTokenAuth from webodm import settings router = routers.DefaultRouter() @@ -63,3 +64,6 @@ if settings.ENABLE_USERS_API: urlpatterns.append(url(r'users', UsersList.as_view())) +if settings.EXTERNAL_AUTH_ENDPOINT != '': + urlpatterns.append(url(r'^external-token-auth/', ExternalTokenAuth.as_view())) + diff --git a/app/auth/backends.py b/app/auth/backends.py index c25f2be3d..22c7a65f3 100644 --- a/app/auth/backends.py +++ b/app/auth/backends.py @@ -8,6 +8,57 @@ logger = logging.getLogger('app.logger') +def get_user_from_external_auth_response(res): + if 'message' in res or 'error' in res: + return None + + if 'user_id' in res and 'username' in res: + try: + user = User.objects.get(pk=res['user_id']) + + # Update user info + if user.username != res['username']: + user.username = res['username'] + user.save() + + # Update quotas + maxQuota = -1 + if 'maxQuota' in res: + maxQuota = res['maxQuota'] + if 'node' in res and 'limits' in res['node'] and 'maxQuota' in res['node']['limits']: + maxQuota = res['node']['limits']['maxQuota'] + + if user.profile.quota != maxQuota: + user.profile.quota = maxQuota + user.save() + except User.DoesNotExist: + user = User(pk=res['user_id'], username=username) + user.save() + + # Setup/update processing node + if ('api_key' in res or 'token' in res) and 'node' in res: + hostname = res['node']['hostname'] + port = res['node']['port'] + token = res['api_key'] if 'api_key' in res else res['token'] + + try: + node = ProcessingNode.objects.get(token=token) + if node.hostname != hostname or node.port != port: + node.hostname = hostname + node.port = port + node.save() + + except ProcessingNode.DoesNotExist: + node = ProcessingNode(hostname=hostname, port=port, token=token) + node.save() + + if not user.has_perm('view_processingnode', node): + assign_perm('view_processingnode', user, node) + + return user + else: + return None + class ExternalBackend(ModelBackend): def authenticate(self, request, username=None, password=None): if EXTERNAL_AUTH_ENDPOINT == "": @@ -20,57 +71,7 @@ def authenticate(self, request, username=None, password=None): }, headers={'Accept': 'application/json'}) res = r.json() - # logger.info(res) - - if 'message' in res or 'error' in res: - return None - - if 'user_id' in res: - try: - user = User.objects.get(pk=res['user_id']) - - # Update user info - if user.username != username: - user.username = username - user.save() - - # Update quotas - maxQuota = -1 - if 'maxQuota' in res: - maxQuota = res['maxQuota'] - if 'node' in res and 'limits' in res['node'] and 'maxQuota' in res['node']['limits']: - maxQuota = res['node']['limits']['maxQuota'] - - if user.profile.quota != maxQuota: - user.profile.quota = maxQuota - user.save() - except User.DoesNotExist: - user = User(pk=res['user_id'], username=username) - user.save() - - # Setup/update processing node - if ('api_key' in res or 'token' in res) and 'node' in res: - hostname = res['node']['hostname'] - port = res['node']['port'] - token = res['api_key'] if 'api_key' in res else res['token'] - - try: - node = ProcessingNode.objects.get(token=token) - if node.hostname != hostname or node.port != port: - node.hostname = hostname - node.port = port - node.save() - - except ProcessingNode.DoesNotExist: - node = ProcessingNode(hostname=hostname, port=port, token=token) - node.save() - - if not user.has_perm('view_processingnode', node): - assign_perm('view_processingnode', user, node) - - return user - else: - return None + return get_user_from_external_auth_response(res) except: return None diff --git a/app/templates/app/registration/login.html b/app/templates/app/registration/login.html index 9945dd422..a1b9b8d43 100644 --- a/app/templates/app/registration/login.html +++ b/app/templates/app/registration/login.html @@ -10,12 +10,12 @@ {% endif %} {% is_single_user_mode as autologin %} - {% external_auth_endpoint as ext_auth_ep %} + {% has_external_auth as ext_auth %} {% if autologin %} {% else %} -
    - {% if ext_auth_ep != '' %} + {% if ext_auth %}
    diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index e439e161c..d904dd1ec 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -10,8 +10,8 @@ logger = logging.getLogger('app.logger') @register.simple_tag -def external_auth_endpoint(): - return settings.EXTERNAL_AUTH_ENDPOINT +def has_external_auth(): + return settings.EXTERNAL_AUTH_ENDPOINT != "" @register.filter def disk_size(megabytes): From 83419a7dab2bb05591e5fbbb610652e8354bd9af Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 8 Sep 2023 15:55:42 -0400 Subject: [PATCH 27/95] Add --settings, drop --external-auth-endpoint --- docker-compose.yml | 2 -- webodm.sh | 18 +++++++++++++----- webodm/settings.py | 8 +++++++- webodm/settings_override.py | 2 ++ 4 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 webodm/settings_override.py diff --git a/docker-compose.yml b/docker-compose.yml index 8fb61b51c..23b8922fd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,6 @@ services: - WO_BROKER - WO_DEV - WO_DEV_WATCH_PLUGINS - - WO_EXTERNAL_AUTH_ENDPOINT restart: unless-stopped oom_score_adj: 0 broker: @@ -53,6 +52,5 @@ services: environment: - WO_BROKER - WO_DEBUG - - WO_EXTERNAL_AUTH_ENDPOINT restart: unless-stopped oom_score_adj: 250 diff --git a/webodm.sh b/webodm.sh index 4a044ba00..5146101ce 100755 --- a/webodm.sh +++ b/webodm.sh @@ -130,9 +130,9 @@ case $key in shift # past argument shift # past value ;; - --external-auth-endpoint) - WO_EXTERNAL_AUTH_ENDPOINT="$2" - export WO_EXTERNAL_AUTH_ENDPOINT + --settings) + WO_SETTINGS=$(realpath "$2") + export WO_SETTINGS shift # past argument shift # past value ;; @@ -176,7 +176,7 @@ usage(){ echo " --broker Set the URL used to connect to the celery broker (default: $DEFAULT_BROKER)" echo " --detached Run WebODM in detached mode. This means WebODM will run in the background, without blocking the terminal (default: disabled)" echo " --gpu Use GPU NodeODM nodes (Linux only) (default: disabled)" - echo " --external-auth-endpoint External authentication endpoint (default: disabled)" + echo " --settings Path to a settings.py file to enable modifications of system settings (default: None)" exit } @@ -346,7 +346,7 @@ start(){ echo "SSL insecure port redirect: $WO_SSL_INSECURE_PORT_REDIRECT" echo "Celery Broker: $WO_BROKER" echo "Default Nodes: $WO_DEFAULT_NODES" - echo "External auth endpoint: $WO_EXTERNAL_AUTH_ENDPOINT" + echo "Settings: $WO_SETTINGS" echo "================================" echo "Make sure to issue a $0 down if you decide to change the environment." echo "" @@ -409,6 +409,14 @@ start(){ echo "Will enable SSL ($method)" fi + if [ ! -z "$WO_SETTINGS" ]; then + if [ ! -e "$WO_SETTINGS" ]; then + echo -e "\033[91mSettings file does not exist: $WO_SETTINGS\033[39m" + exit 1 + fi + command+=" -f docker-compose.settings.yml" + fi + command="$command up" if [[ $detached = true ]]; then diff --git a/webodm/settings.py b/webodm/settings.py index 9774338f8..a09aba700 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -391,7 +391,8 @@ def scalebyiv(color, n): # before it should be considered offline NODE_OFFLINE_MINUTES = 5 -EXTERNAL_AUTH_ENDPOINT = os.environ.get('WO_EXTERNAL_AUTH_ENDPOINT', '') +EXTERNAL_AUTH_ENDPOINT = '' +RESET_PASSWORD_LINK = '' # Number of hours before tasks are automatically deleted # from an account that is exceeding a disk quota @@ -405,3 +406,8 @@ def scalebyiv(color, n): from .local_settings import * except ImportError: pass + +try: + from .settings_override import * +except ImportError: + pass \ No newline at end of file diff --git a/webodm/settings_override.py b/webodm/settings_override.py new file mode 100644 index 000000000..79c0d64ec --- /dev/null +++ b/webodm/settings_override.py @@ -0,0 +1,2 @@ +# Do not touch. This file can be bind-mount replaced +# by docker-compose for customized settings \ No newline at end of file From e7d57b4cd58099dd567005308bf7ba057020a4e2 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 8 Sep 2023 15:56:05 -0400 Subject: [PATCH 28/95] Add docker-compose file --- docker-compose.settings.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docker-compose.settings.yml diff --git a/docker-compose.settings.yml b/docker-compose.settings.yml new file mode 100644 index 000000000..16182fb1c --- /dev/null +++ b/docker-compose.settings.yml @@ -0,0 +1,8 @@ +version: '2.1' +services: + webapp: + volumes: + - ${WO_SETTINGS}:/webodm/webodm/settings_override.py + worker: + volumes: + - ${WO_SETTINGS}:/webodm/webodm/settings_override.py \ No newline at end of file From 54296bd7a4ee441a91b224cf04804df3d3b16330 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 8 Sep 2023 16:02:45 -0400 Subject: [PATCH 29/95] Read pwd reset link from settings --- app/templates/app/registration/login.html | 8 ++++++-- app/templatetags/settings.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/app/templates/app/registration/login.html b/app/templates/app/registration/login.html index a1b9b8d43..c19260f2a 100644 --- a/app/templates/app/registration/login.html +++ b/app/templates/app/registration/login.html @@ -11,6 +11,7 @@ {% is_single_user_mode as autologin %} {% has_external_auth as ext_auth %} + {% reset_password_link as reset_pwd_link %} {% if autologin %} @@ -24,14 +25,17 @@
    - -

    Forgot your password?

    + {% if reset_pwd_link != '' %} +

    {% trans "Forgot your password?" %}

    + {% else %} +

    {% trans "Forgot your password?" %}

    + {% endif %}
    diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index d904dd1ec..96efd0e21 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -9,6 +9,10 @@ register = template.Library() logger = logging.getLogger('app.logger') +@register.simple_tag +def reset_password_link(): + return settings.RESET_PASSWORD_LINK + @register.simple_tag def has_external_auth(): return settings.EXTERNAL_AUTH_ENDPOINT != "" From 039df51cc6985e4e77a397d572fdc6b8e2c76dd3 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 8 Sep 2023 17:38:05 -0400 Subject: [PATCH 30/95] Add some unit tests --- .env | 2 +- app/api/admin.py | 1 + app/api/externalauth.py | 29 ++++++++++++++ app/tests/test_api_admin.py | 55 ++++++++++++++++++++++++++ app/tests/test_external_auth.py | 68 +++++++++++++++++++++++++++++++++ webodm/settings.py | 1 + 6 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 app/tests/test_external_auth.py diff --git a/.env b/.env index 516bcd214..b417a4900 100644 --- a/.env +++ b/.env @@ -10,4 +10,4 @@ WO_DEBUG=NO WO_DEV=NO WO_BROKER=redis://broker WO_DEFAULT_NODES=1 -WO_EXTERNAL_AUTH_ENDPOINT= +WO_SETTINGS= diff --git a/app/api/admin.py b/app/api/admin.py index 2de55e4dd..15e136a28 100644 --- a/app/api/admin.py +++ b/app/api/admin.py @@ -58,6 +58,7 @@ class Meta: read_only_fields = ('user', ) class AdminProfileViewSet(viewsets.ModelViewSet): + pagination_class = None serializer_class = ProfileSerializer permission_classes = [IsAdminUser] lookup_field = 'user' diff --git a/app/api/externalauth.py b/app/api/externalauth.py index dcc96a609..8a77c71a0 100644 --- a/app/api/externalauth.py +++ b/app/api/externalauth.py @@ -36,3 +36,32 @@ def post(self, request): return Response({'error': res.get('message', 'Invalid external server response')}) except Exception as e: return Response({'error': str(e)}) + +# TODO: move to simple http server +# class TestExternalAuth(APIView): +# permission_classes = (permissions.AllowAny,) +# parser_classes = (parsers.JSONParser, parsers.FormParser,) + +# def post(self, request): +# print("YO!!!") +# if settings.EXTERNAL_AUTH_ENDPOINT == '': +# return Response({'message': 'Disabled'}) + +# username = request.data.get("username") +# password = request.data.get("password") + +# print("HERE", username) + +# if username == "extuser1" and password == "test1234": +# return Response({ +# 'user_id': 100, +# 'username': 'extuser1', +# 'maxQuota': 500, +# 'token': 'test', +# 'node': { +# 'hostname': 'localhost', +# 'port': 4444 +# } +# }) +# else: +# return Response({'message': "Invalid credentials"}) \ No newline at end of file diff --git a/app/tests/test_api_admin.py b/app/tests/test_api_admin.py index 7ba0fa282..7a0d1f7f0 100644 --- a/app/tests/test_api_admin.py +++ b/app/tests/test_api_admin.py @@ -1,3 +1,4 @@ +import time from django.contrib.auth.models import User, Group from rest_framework import status from rest_framework.test import APIClient @@ -202,3 +203,57 @@ def test_group(self): res = client.delete('/api/admin/groups/{}/'.format(group.id)) self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + def test_profile(self): + client = APIClient() + client.login(username="testuser", password="test1234") + + user = User.objects.get(username="testuser") + + # Cannot list profiles (not admin) + res = client.get('/api/admin/profiles/') + self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + + res = client.get('/api/admin/profiles/%s/' % user.id) + self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + + # Cannot update quota deadlines + res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id, data={'hours': 1}) + self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + + # Admin can + client.login(username="testsuperuser", password="test1234") + + res = client.get('/api/admin/profiles/') + self.assertEqual(res.status_code, status.HTTP_200_OK) + self.assertTrue(len(res.data) > 0) + + res = client.get('/api/admin/profiles/%s/' % user.id) + self.assertEqual(res.status_code, status.HTTP_200_OK) + self.assertTrue('quota' in res.data) + self.assertTrue('user' in res.data) + + # User is the primary key (not profile id) + self.assertEqual(res.data['user'], user.id) + + # There should be no quota by default + self.assertEqual(res.data['quota'], -1) + + # Try updating + user.profile.quota = 10 + user.save() + res = client.get('/api/admin/profiles/%s/' % user.id) + self.assertEqual(res.data['quota'], 10) + + # Update quota deadlines + + # Miss parameters + res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id) + self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST) + + res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id, data={'hours': 48}) + self.assertEqual(res.status_code, status.HTTP_200_OK) + self.assertTrue('deadline' in res.data and res.data['deadline'] > time.time() + 47*60*60) + + res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id, data={'hours': 0}) + self.assertEqual(res.status_code, status.HTTP_200_OK) + self.assertTrue(abs(user.profile.get_quota_deadline() - time.time()) < 10) diff --git a/app/tests/test_external_auth.py b/app/tests/test_external_auth.py new file mode 100644 index 000000000..90dcda3f5 --- /dev/null +++ b/app/tests/test_external_auth.py @@ -0,0 +1,68 @@ +from django.contrib.auth.models import User, Group +from rest_framework import status +from rest_framework.test import APIClient + +from .classes import BootTestCase +from webodm import settings + +class TestAuth(BootTestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_ext_auth(self): + client = APIClient() + + # Disable + settings.EXTERNAL_AUTH_ENDPOINT = '' + + # Try to log-in + user = client.login(username='extuser1', password='test1234') + self.assertFalse(user) + + # Enable + settings.EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555' + + # TODO: start simplehttp auth server + + user = client.login(username='extuser1', password='test1234') + # self.assertEqual(user.username, 'extuser1') + # self.assertEqual(user.id, 100) + + + # client.login(username="testuser", password="test1234") + + # user = User.objects.get(username="testuser") + + # # Cannot list profiles (not admin) + # res = client.get('/api/admin/profiles/') + # self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + + # res = client.get('/api/admin/profiles/%s/' % user.id) + # self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + + # # Cannot update quota deadlines + # res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id, data={'hours': 1}) + # self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) + + # # Admin can + # client.login(username="testsuperuser", password="test1234") + + # res = client.get('/api/admin/profiles/') + # self.assertEqual(res.status_code, status.HTTP_200_OK) + # self.assertTrue(len(res.data) > 0) + + # res = client.get('/api/admin/profiles/%s/' % user.id) + # self.assertEqual(res.status_code, status.HTTP_200_OK) + # self.assertTrue('quota' in res.data) + # self.assertTrue('user' in res.data) + + # # User is the primary key (not profile id) + # self.assertEqual(res.data['user'], user.id) + + # # There should be no quota by default + # self.assertEqual(res.data['quota'], -1) + + \ No newline at end of file diff --git a/webodm/settings.py b/webodm/settings.py index a09aba700..74f79cf06 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -401,6 +401,7 @@ def scalebyiv(color, n): if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True + EXTERNAL_AUTH_ENDPOINT = '/_test-external-auth' try: from .local_settings import * From a709c8fdf6e7519618605eed7567e4c8bc04b72b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 11 Sep 2023 11:53:10 -0400 Subject: [PATCH 31/95] Moar tests --- app/api/externalauth.py | 28 ------- app/auth/backends.py | 69 +++++++++--------- app/tests/scripts/simple_auth_server.py | 97 +++++++++++++++++++++++++ app/tests/test_external_auth.py | 70 +++++++----------- app/tests/test_quota.py | 58 +++++++++++++++ app/tests/utils.py | 10 +++ nodeodm/models.py | 4 + webodm/settings.py | 2 +- 8 files changed, 231 insertions(+), 107 deletions(-) create mode 100644 app/tests/scripts/simple_auth_server.py create mode 100644 app/tests/test_quota.py diff --git a/app/api/externalauth.py b/app/api/externalauth.py index 8a77c71a0..5f9c564c7 100644 --- a/app/api/externalauth.py +++ b/app/api/externalauth.py @@ -37,31 +37,3 @@ def post(self, request): except Exception as e: return Response({'error': str(e)}) -# TODO: move to simple http server -# class TestExternalAuth(APIView): -# permission_classes = (permissions.AllowAny,) -# parser_classes = (parsers.JSONParser, parsers.FormParser,) - -# def post(self, request): -# print("YO!!!") -# if settings.EXTERNAL_AUTH_ENDPOINT == '': -# return Response({'message': 'Disabled'}) - -# username = request.data.get("username") -# password = request.data.get("password") - -# print("HERE", username) - -# if username == "extuser1" and password == "test1234": -# return Response({ -# 'user_id': 100, -# 'username': 'extuser1', -# 'maxQuota': 500, -# 'token': 'test', -# 'node': { -# 'hostname': 'localhost', -# 'port': 4444 -# } -# }) -# else: -# return Response({'message': "Invalid credentials"}) \ No newline at end of file diff --git a/app/auth/backends.py b/app/auth/backends.py index 22c7a65f3..c3c13ea14 100644 --- a/app/auth/backends.py +++ b/app/auth/backends.py @@ -2,7 +2,7 @@ from django.contrib.auth.backends import ModelBackend from django.contrib.auth.models import User from nodeodm.models import ProcessingNode -from webodm.settings import EXTERNAL_AUTH_ENDPOINT +from webodm import settings from guardian.shortcuts import assign_perm import logging @@ -15,45 +15,48 @@ def get_user_from_external_auth_response(res): if 'user_id' in res and 'username' in res: try: user = User.objects.get(pk=res['user_id']) + except User.DoesNotExist: + user = User(pk=res['user_id'], username=res['username']) + user.save() - # Update user info - if user.username != res['username']: - user.username = res['username'] - user.save() - - # Update quotas - maxQuota = -1 - if 'maxQuota' in res: - maxQuota = res['maxQuota'] - if 'node' in res and 'limits' in res['node'] and 'maxQuota' in res['node']['limits']: - maxQuota = res['node']['limits']['maxQuota'] + # Update user info + if user.username != res['username']: + user.username = res['username'] + user.save() + + maxQuota = -1 + if 'maxQuota' in res: + maxQuota = res['maxQuota'] + if 'node' in res and 'limits' in res['node'] and 'maxQuota' in res['node']['limits']: + maxQuota = res['node']['limits']['maxQuota'] - if user.profile.quota != maxQuota: - user.profile.quota = maxQuota - user.save() - except User.DoesNotExist: - user = User(pk=res['user_id'], username=username) + # Update quotas + if user.profile.quota != maxQuota: + user.profile.quota = maxQuota user.save() # Setup/update processing node - if ('api_key' in res or 'token' in res) and 'node' in res: + if 'node' in res and 'hostname' in res['node'] and 'port' in res['node']: hostname = res['node']['hostname'] port = res['node']['port'] - token = res['api_key'] if 'api_key' in res else res['token'] + token = res['node'].get('token', '') - try: - node = ProcessingNode.objects.get(token=token) - if node.hostname != hostname or node.port != port: - node.hostname = hostname - node.port = port + # Only add/update if a token is provided, since we use + # tokens as unique identifiers for hostname/port updates + if token != "": + try: + node = ProcessingNode.objects.get(token=token) + if node.hostname != hostname or node.port != port: + node.hostname = hostname + node.port = port + node.save() + + except ProcessingNode.DoesNotExist: + node = ProcessingNode(hostname=hostname, port=port, token=token) node.save() - except ProcessingNode.DoesNotExist: - node = ProcessingNode(hostname=hostname, port=port, token=token) - node.save() - - if not user.has_perm('view_processingnode', node): - assign_perm('view_processingnode', user, node) + if not user.has_perm('view_processingnode', node): + assign_perm('view_processingnode', user, node) return user else: @@ -61,11 +64,11 @@ def get_user_from_external_auth_response(res): class ExternalBackend(ModelBackend): def authenticate(self, request, username=None, password=None): - if EXTERNAL_AUTH_ENDPOINT == "": + if settings.EXTERNAL_AUTH_ENDPOINT == "": return None try: - r = requests.post(EXTERNAL_AUTH_ENDPOINT, { + r = requests.post(settings.EXTERNAL_AUTH_ENDPOINT, { 'username': username, 'password': password }, headers={'Accept': 'application/json'}) @@ -76,7 +79,7 @@ def authenticate(self, request, username=None, password=None): return None def get_user(self, user_id): - if EXTERNAL_AUTH_ENDPOINT == "": + if settings.EXTERNAL_AUTH_ENDPOINT == "": return None try: diff --git a/app/tests/scripts/simple_auth_server.py b/app/tests/scripts/simple_auth_server.py new file mode 100644 index 000000000..690b86366 --- /dev/null +++ b/app/tests/scripts/simple_auth_server.py @@ -0,0 +1,97 @@ +import http.server +from http.server import SimpleHTTPRequestHandler +import socketserver +import sys +import threading +from time import sleep +import json + +class MyHandler(SimpleHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type','text/html') + self.end_headers() + self.wfile.write(bytes("Simple auth server is running", encoding="utf-8")) + + + def send_error(self, code, error): + self.send_json(code, {"error": error}) + + def send_json(self, code, data): + response = bytes(json.dumps(data), encoding="utf-8") + + self.send_response(200) + self.send_header('Content-type','application/json') + self.send_header('Content-length', len(response)) + self.end_headers() + self.wfile.write(response) + + def do_POST(self): + if self.path == '/auth': + if not 'Content-Length' in self.headers: + self.send_error(403, "Missing form data") + return + + content_length = int(self.headers['Content-Length']) + post_data_str = self.rfile.read(content_length).decode("utf-8") + post_data = {} + for item in post_data_str.split('&'): + k,v = item.split('=') + post_data[k] = v + + username = post_data.get("username") + password = post_data.get("password") + + print("Login request for " + username) + + if username == "extuser1" and password == "test1234": + print("Granted") + self.send_json(200, { + 'user_id': 100, + 'username': 'extuser1', + 'maxQuota': 500, + 'node': { + 'hostname': 'localhost', + 'port': 4444, + 'token': 'test' + } + }) + else: + print("Unauthorized") + return self.send_error(401, "unauthorized") + else: + self.send_error(404, "not found") + +class WebServer(threading.Thread): + def __init__(self): + super().__init__() + self.host = "0.0.0.0" + self.port = int(sys.argv[1]) if len(sys.argv) >= 2 else 8080 + self.ws = socketserver.TCPServer((self.host, self.port), MyHandler) + + def run(self): + print("WebServer started at Port:", self.port) + self.ws.serve_forever() + + def shutdown(self): + # set the two flags needed to shutdown the HTTP server manually + # self.ws._BaseServer__is_shut_down.set() + # self.ws.__shutdown_request = True + + print('Shutting down server.') + # call it anyway, for good measure... + self.ws.shutdown() + print('Closing server.') + self.ws.server_close() + self.join() + +if __name__=='__main__': + webServer = WebServer() + webServer.start() + while True: + try: + sleep(0.5) + except KeyboardInterrupt: + print('Keyboard Interrupt sent.') + webServer.shutdown() + exit(0) \ No newline at end of file diff --git a/app/tests/test_external_auth.py b/app/tests/test_external_auth.py index 90dcda3f5..2928bebd7 100644 --- a/app/tests/test_external_auth.py +++ b/app/tests/test_external_auth.py @@ -1,8 +1,10 @@ from django.contrib.auth.models import User, Group +from nodeodm.models import ProcessingNode from rest_framework import status from rest_framework.test import APIClient from .classes import BootTestCase +from .utils import start_simple_auth_server from webodm import settings class TestAuth(BootTestCase): @@ -19,50 +21,28 @@ def test_ext_auth(self): settings.EXTERNAL_AUTH_ENDPOINT = '' # Try to log-in - user = client.login(username='extuser1', password='test1234') - self.assertFalse(user) + ok = client.login(username='extuser1', password='test1234') + self.assertFalse(ok) # Enable - settings.EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555' - - # TODO: start simplehttp auth server - - user = client.login(username='extuser1', password='test1234') - # self.assertEqual(user.username, 'extuser1') - # self.assertEqual(user.id, 100) - - - # client.login(username="testuser", password="test1234") - - # user = User.objects.get(username="testuser") - - # # Cannot list profiles (not admin) - # res = client.get('/api/admin/profiles/') - # self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) - - # res = client.get('/api/admin/profiles/%s/' % user.id) - # self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) - - # # Cannot update quota deadlines - # res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id, data={'hours': 1}) - # self.assertEqual(res.status_code, status.HTTP_403_FORBIDDEN) - - # # Admin can - # client.login(username="testsuperuser", password="test1234") - - # res = client.get('/api/admin/profiles/') - # self.assertEqual(res.status_code, status.HTTP_200_OK) - # self.assertTrue(len(res.data) > 0) - - # res = client.get('/api/admin/profiles/%s/' % user.id) - # self.assertEqual(res.status_code, status.HTTP_200_OK) - # self.assertTrue('quota' in res.data) - # self.assertTrue('user' in res.data) - - # # User is the primary key (not profile id) - # self.assertEqual(res.data['user'], user.id) - - # # There should be no quota by default - # self.assertEqual(res.data['quota'], -1) - - \ No newline at end of file + settings.EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555/auth' + + with start_simple_auth_server(["5555"]): + ok = client.login(username='extuser1', password='invalid') + self.assertFalse(ok) + self.assertFalse(User.objects.filter(username="extuser1").exists()) + ok = client.login(username='extuser1', password='test1234') + self.assertTrue(ok) + user = User.objects.get(username="extuser1") + self.assertEqual(user.id, 100) + self.assertEqual(user.profile.quota, 500) + pnode = ProcessingNode.objects.get(token='test') + self.assertEqual(pnode.hostname, 'localhost') + self.assertEqual(pnode.port, 4444) + self.assertTrue(user.has_perm('view_processingnode', pnode)) + self.assertFalse(user.has_perm('delete_processingnode', pnode)) + self.assertFalse(user.has_perm('change_processingnode', pnode)) + + # Re-test login + ok = client.login(username='extuser1', password='test1234') + self.assertTrue(ok) diff --git a/app/tests/test_quota.py b/app/tests/test_quota.py new file mode 100644 index 000000000..0e43b33a2 --- /dev/null +++ b/app/tests/test_quota.py @@ -0,0 +1,58 @@ +from django.contrib.auth.models import User, Group +from rest_framework import status +from rest_framework.test import APIClient +from app.models import Task, Project +from .classes import BootTestCase + +class TestQuota(BootTestCase): + def setUp(self): + pass + + def tearDown(self): + pass + + def test_quota(self): + c = APIClient() + c.login(username="testuser", password="test1234") + + user = User.objects.get(username="testuser") + self.assertEqual(user.profile.quota, -1) + + # There should be no quota panel + res = c.get('/dashboard/', follow=True) + body = res.content.decode("utf-8") + + # There should be no quota panel + self.assertFalse('
    ' in body) + + user.profile.quota = 2000 + user.save() + + res = c.get('/dashboard/', follow=True) + body = res.content.decode("utf-8") + + # There should be a quota panel + self.assertTrue('
    ' in body) + + # There should be no warning + self.assertFalse("disk quota is being exceeded" in body) + + self.assertEqual(user.profile.used_quota(), 0) + self.assertEqual(user.profile.used_quota_cached(), 0) + + # Create a task with size + p = Project.objects.create(owner=user, name='Test') + p.save() + t = Task.objects.create(project=p, name='Test', size=2005) + t.save() + + # Simulate call to task.update_size which calls clear_used_quota_cache + user.profile.clear_used_quota_cache() + + self.assertTrue(user.profile.has_exceeded_quota()) + self.assertTrue(user.profile.has_exceeded_quota_cached()) + + res = c.get('/dashboard/', follow=True) + body = res.content.decode("utf-8") + + # self.assertTrue("disk quota is being exceeded" in body) diff --git a/app/tests/utils.py b/app/tests/utils.py index f5ca14561..763546c74 100644 --- a/app/tests/utils.py +++ b/app/tests/utils.py @@ -25,6 +25,16 @@ def start_processing_node(args = []): node_odm.terminate() time.sleep(1) # Wait for the server to stop +@contextmanager +def start_simple_auth_server(args = []): + current_dir = os.path.dirname(os.path.realpath(__file__)) + s = subprocess.Popen(['python', 'simple_auth_server.py'] + args, shell=False, + cwd=os.path.join(current_dir, "scripts")) + time.sleep(2) # Wait for the server to launch + yield s + s.terminate() + time.sleep(1) # Wait for the server to stop + # We need to clear previous media_root content # This points to the test directory, but just in case # we double check that the directory is indeed a test directory diff --git a/nodeodm/models.py b/nodeodm/models.py index 39f47af89..a2f8e81e3 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -15,7 +15,9 @@ from pyodm import exceptions from django.db.models import signals from datetime import timedelta +import logging +logger = logging.getLogger('app.logger') class ProcessingNode(models.Model): hostname = models.CharField(verbose_name=_("Hostname"), max_length=255, help_text=_("Hostname or IP address where the node is located (can be an internal hostname as well). If you are using Docker, this is never 127.0.0.1 or localhost. Find the IP address of your host machine by running ifconfig on Linux or by checking your network settings.")) @@ -197,6 +199,8 @@ def auto_update_node_info(sender, instance, created, **kwargs): instance.update_node_info() except exceptions.OdmError: pass + except Exception as e: + logger.warning("auto_update_node_info: " + str(e)) class ProcessingNodeUserObjectPermission(UserObjectPermissionBase): content_object = models.ForeignKey(ProcessingNode, on_delete=models.CASCADE) diff --git a/webodm/settings.py b/webodm/settings.py index 74f79cf06..04389728e 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -401,7 +401,7 @@ def scalebyiv(color, n): if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True - EXTERNAL_AUTH_ENDPOINT = '/_test-external-auth' + EXTERNAL_AUTH_ENDPOINT = 'http://0.0.0.0:5555/auth' try: from .local_settings import * From c7ff74a5266a84dfdd61d22309ef0ad5c9eb4abb Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 11 Sep 2023 13:02:46 -0400 Subject: [PATCH 32/95] Moar unit tests --- app/api/tasks.py | 1 + app/templates/app/dashboard.html | 13 ++++------ app/templates/app/logged_in_base.html | 2 +- app/templates/app/quota.html | 11 +++++++++ app/templatetags/settings.py | 8 +++---- app/tests/test_api_admin.py | 2 ++ app/tests/test_login.py | 34 +++++++++++++++++++++++++++ app/tests/test_quota.py | 34 +++++++++++++++++++++++++-- 8 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 app/templates/app/quota.html create mode 100644 app/tests/test_login.py diff --git a/app/api/tasks.py b/app/api/tasks.py index 4e4da2da6..bb2d4a7cb 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -184,6 +184,7 @@ def commit(self, request, pk=None, project_pk=None): if task.images_count < 1: raise exceptions.ValidationError(detail=_("You need to upload at least 1 file before commit")) + task.update_size() task.save() worker_tasks.process_task.delay(task.id) diff --git a/app/templates/app/dashboard.html b/app/templates/app/dashboard.html index 668af63e4..344981ad2 100644 --- a/app/templates/app/dashboard.html +++ b/app/templates/app/dashboard.html @@ -13,6 +13,8 @@ {% if no_processingnodes %} + {% include "quota.html" %} +

    {% trans 'Welcome!' %} ☺

    {% trans 'Add a Processing Node' as add_processing_node %} {% with nodeodm_link='NodeODM' api_link='API' %} @@ -39,15 +41,8 @@

    {% trans 'Welcome!' %} ☺

    {% endif %} - - {% if user.profile.has_exceeded_quota_cached %} - {% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %} - {% quota_exceeded_grace_period as when %} -
    - {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted {{ when }}, until usage falls below {{ total }}.{% endblocktrans %} -
    - {% endwith %} - {% endif %} + + {% include "quota.html" %}
    diff --git a/app/templates/app/logged_in_base.html b/app/templates/app/logged_in_base.html index 8ba8d36f6..914cb4b07 100644 --- a/app/templates/app/logged_in_base.html +++ b/app/templates/app/logged_in_base.html @@ -19,7 +19,7 @@
    • {% if user.profile.has_quota %}
  • - + {% with tot_quota=user.profile.quota used_quota=user.profile.used_quota_cached %} {% percentage used_quota tot_quota as perc_quota %} {% percentage used_quota tot_quota 100 as bar_width %} diff --git a/app/templates/app/quota.html b/app/templates/app/quota.html new file mode 100644 index 000000000..c52df023a --- /dev/null +++ b/app/templates/app/quota.html @@ -0,0 +1,11 @@ +{% load i18n %} +{% load settings %} + +{% if user.profile.has_exceeded_quota_cached %} + {% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %} + {% quota_exceeded_grace_period as when %} +
    + {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted {{ when }}, until usage falls below {{ total }}.{% endblocktrans %} +
    + {% endwith %} +{% endif %} \ No newline at end of file diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index 96efd0e21..b0962cbd2 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -46,11 +46,11 @@ def quota_exceeded_grace_period(context): deadline = now + settings.QUOTA_EXCEEDED_GRACE_PERIOD * 60 * 60 diff = max(0, deadline - now) if diff >= 60*60*24*2: - return _("within %(num)s days") % {"num": math.ceil(diff / (60*60*24))} - elif diff >= 60*60: - return _("within %(num)s hours") % {"num": math.ceil(diff / (60*60))} + return _("in %(num)s days") % {"num": math.floor(diff / (60*60*24))} + elif diff >= 60*60*2: + return _("in %(num)s hours") % {"num": math.floor(diff / (60*60))} elif diff > 1: - return _("within %(num)s minutes") % {"num": math.ceil(diff / 60)} + return _("in %(num)s minutes") % {"num": math.floor(diff / 60)} else: return _("very soon") diff --git a/app/tests/test_api_admin.py b/app/tests/test_api_admin.py index 7a0d1f7f0..a2c46b385 100644 --- a/app/tests/test_api_admin.py +++ b/app/tests/test_api_admin.py @@ -246,6 +246,8 @@ def test_profile(self): # Update quota deadlines + self.assertTrue(user.profile.get_quota_deadline() is None) + # Miss parameters res = client.post('/api/admin/profiles/%s/update_quota_deadline/' % user.id) self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/app/tests/test_login.py b/app/tests/test_login.py new file mode 100644 index 000000000..57f25a15c --- /dev/null +++ b/app/tests/test_login.py @@ -0,0 +1,34 @@ +import os +from django.test import Client +from webodm import settings +from .classes import BootTestCase + +class TestLogin(BootTestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_reset_password_render(self): + c = Client() + c.login(username="testuser", password="test1234") + + settings.RESET_PASSWORD_LINK = '' + + res = c.get('/login/', follow=True) + body = res.content.decode("utf-8") + + # The reset password link should show instructions + self.assertTrue("You can reset the administrator password" in body) + + settings.RESET_PASSWORD_LINK = 'http://0.0.0.0/reset_test' + + res = c.get('/login/', follow=True) + body = res.content.decode("utf-8") + + # The reset password link should show instructions + self.assertTrue(' Date: Mon, 11 Sep 2023 13:48:29 -0400 Subject: [PATCH 33/95] Update locales --- .../app/js/translations/odm_autogenerated.js | 156 +++++++++--------- app/tests/test_login.py | 2 +- .../plugin_manifest_autogenerated.py | 9 +- locale | 2 +- 4 files changed, 85 insertions(+), 84 deletions(-) diff --git a/app/static/app/js/translations/odm_autogenerated.js b/app/static/app/js/translations/odm_autogenerated.js index bc6021da4..d4f81208f 100644 --- a/app/static/app/js/translations/odm_autogenerated.js +++ b/app/static/app/js/translations/odm_autogenerated.js @@ -1,94 +1,94 @@ // Auto-generated with extract_odm_strings.py, do not edit! -_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); -_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); -_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); -_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); -_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); -_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); +_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); +_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); +_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); +_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); +_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); +_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); _("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); -_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); +_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); +_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); +_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); +_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); +_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); +_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); +_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); _("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); -_("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); +_("Generate OGC 3D Tiles outputs. Default: %(default)s"); +_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); +_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); +_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); _("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); -_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); -_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); -_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); -_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); -_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); -_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); -_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); -_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); -_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); -_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); +_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); _("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); -_("show this help message and exit"); -_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); -_("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); -_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); -_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); -_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); -_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); -_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); -_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); -_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); -_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); -_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); +_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); _("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); -_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); -_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); -_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); -_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); -_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); -_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); -_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); -_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); -_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); -_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); -_("Permanently delete all previous results and rerun the processing pipeline."); -_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); _("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); -_("DSM/DTM resolution in cm / pixel. Note that this value is capped to 2x the ground sampling distance (GSD) estimate. To remove the cap, check --ignore-gsd also. Default: %(default)s"); -_("Displays version number and exits. "); -_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); -_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); -_("Skip the blending of colors near seams. Default: %(default)s"); -_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); -_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); -_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); +_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); +_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); +_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); +_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); +_("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); _("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); -_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. To remove the cap, check --ignore-gsd also. Default: %(default)s"); -_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); -_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); -_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); -_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); -_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); -_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); -_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); -_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); +_("The maximum vertex count of the output mesh. Default: %(default)s"); +_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); +_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); +_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); _("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); +_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); +_("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); _("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); -_("Copy output results to this folder after processing."); -_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); -_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); +_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); +_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); +_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); +_("show this help message and exit"); _("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); -_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); -_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); +_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); _("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); +_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); +_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); +_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); +_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); +_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); +_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); +_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); +_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); +_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); _("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); -_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); -_("Generate OGC 3D Tiles outputs. Default: %(default)s"); -_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); -_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); -_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); +_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); +_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); +_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); _("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); -_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); -_("Ignore Ground Sampling Distance (GSD). GSD caps the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Default: %(default)s"); -_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. Set to 0 to disable. Default: %(default)s"); -_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); -_("The maximum vertex count of the output mesh. Default: %(default)s"); -_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); -_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); -_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); +_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); +_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); +_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); +_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); +_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); +_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); +_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); +_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); +_("Displays version number and exits. "); +_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); _("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); +_("Permanently delete all previous results and rerun the processing pipeline."); +_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); +_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); +_("Copy output results to this folder after processing."); +_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); +_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); +_("Skip the blending of colors near seams. Default: %(default)s"); diff --git a/app/tests/test_login.py b/app/tests/test_login.py index 57f25a15c..371e3bcff 100644 --- a/app/tests/test_login.py +++ b/app/tests/test_login.py @@ -28,7 +28,7 @@ def test_reset_password_render(self): res = c.get('/login/', follow=True) body = res.content.decode("utf-8") - # The reset password link should show instructions + # The reset password link is a link self.assertTrue(' Date: Mon, 11 Sep 2023 16:35:54 -0400 Subject: [PATCH 34/95] Move task.console_output --- app/api/tasks.py | 9 ++-- app/classes/console.py | 48 +++++++++++++++++++ .../0038_remove_task_console_output.py | 42 ++++++++++++++++ app/models/task.py | 22 ++++++--- app/tests/test_api.py | 6 +-- coreplugins/cloudimport/api_views.py | 2 +- coreplugins/dronedb/api_views.py | 2 +- coreplugins/tasknotification/signals.py | 6 +-- package.json | 2 +- 9 files changed, 118 insertions(+), 21 deletions(-) create mode 100644 app/classes/console.py create mode 100644 app/migrations/0038_remove_task_console_output.py diff --git a/app/api/tasks.py b/app/api/tasks.py index bb2d4a7cb..191178b6f 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -74,7 +74,7 @@ def get_can_rerun_from(self, obj): class Meta: model = models.Task - exclude = ('console_output', 'orthophoto_extent', 'dsm_extent', 'dtm_extent', ) + exclude = ('orthophoto_extent', 'dsm_extent', 'dtm_extent', ) read_only_fields = ('processing_time', 'status', 'last_error', 'created_at', 'pending_action', 'available_assets', 'size', ) class TaskViewSet(viewsets.ViewSet): @@ -83,7 +83,7 @@ class TaskViewSet(viewsets.ViewSet): A task represents a set of images and other input to be sent to a processing node. Once a processing node completes processing, results are stored in the task. """ - queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', 'console_output', ) + queryset = models.Task.objects.all().defer('orthophoto_extent', 'dsm_extent', 'dtm_extent', ) parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser, ) ordering_fields = '__all__' @@ -145,8 +145,7 @@ def output(self, request, pk=None, project_pk=None): raise exceptions.NotFound() line_num = max(0, int(request.query_params.get('line', 0))) - output = task.console_output or "" - return Response('\n'.join(output.rstrip().split('\n')[line_num:])) + return Response('\n'.join(task.console.output().rstrip().split('\n')[line_num:])) def list(self, request, project_pk=None): get_and_check_project(request, project_pk) @@ -296,7 +295,7 @@ def partial_update(self, request, *args, **kwargs): class TaskNestedView(APIView): - queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', ) + queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', ) permission_classes = (AllowAny, ) def get_and_check_task(self, request, pk, annotate={}): diff --git a/app/classes/console.py b/app/classes/console.py new file mode 100644 index 000000000..d9f0a1e25 --- /dev/null +++ b/app/classes/console.py @@ -0,0 +1,48 @@ +import os +import logging +logger = logging.getLogger('app.logger') + +class Console: + def __init__(self, file): + self.file = file + self.base_dir = os.path.dirname(self.file) + self.parent_dir = os.path.dirname(self.base_dir) + + def __repr__(self): + return "" % self.file + + def __str__(self): + if not os.path.isfile(self.file): + return "" + + try: + with open(self.file, 'r') as f: + return f.read() + except IOError: + logger.warn("Cannot read console file: %s" % self.file) + return "" + + def __add__(self, other): + self.append(other) + return self + + def output(self): + return str(self) + + def append(self, text): + if os.path.isdir(self.parent_dir): + # Write + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir, exist_ok=True) + + with open(self.file, "a") as f: + f.write(text) + + def reset(self, text = ""): + if os.path.isdir(self.parent_dir): + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir, exist_ok=True) + + with open(self.file, "w") as f: + f.write(text) + diff --git a/app/migrations/0038_remove_task_console_output.py b/app/migrations/0038_remove_task_console_output.py new file mode 100644 index 000000000..88fc024f2 --- /dev/null +++ b/app/migrations/0038_remove_task_console_output.py @@ -0,0 +1,42 @@ +# Generated by Django 2.2.27 on 2023-09-11 19:11 +import os +from django.db import migrations +from webodm import settings + +def data_path(project_id, task_id, *args): + return os.path.join(settings.MEDIA_ROOT, + "project", + str(project_id), + "task", + str(task_id), + "data", + *args) + +def dump_console_outputs(apps, schema_editor): + Task = apps.get_model('app', 'Task') + + for t in Task.objects.all(): + if t.console_output is not None and len(t.console_output) > 0: + dp = data_path(t.project.id, t.id) + os.makedirs(dp, exist_ok=True) + outfile = os.path.join(dp, "console_output.txt") + + with open(outfile, "w") as f: + f.write(t.console_output) + print("Wrote console output for %s to %s" % (t, outfile)) + else: + print("No task output for %s" % t) + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0037_profile'), + ] + + operations = [ + migrations.RunPython(dump_console_outputs), + migrations.RemoveField( + model_name='task', + name='console_output', + ), + ] diff --git a/app/models/task.py b/app/models/task.py index 4235011cf..1f0c4c0c6 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -46,6 +46,7 @@ from functools import partial import subprocess +from app.classes.console import Console logger = logging.getLogger('app.logger') @@ -247,7 +248,6 @@ class Task(models.Model): last_error = models.TextField(null=True, blank=True, help_text=_("The last processing error received"), verbose_name=_("Last Error")) options = fields.JSONField(default=dict, blank=True, help_text=_("Options that are being used to process this task"), validators=[validate_task_options], verbose_name=_("Options")) available_assets = fields.ArrayField(models.CharField(max_length=80), default=list, blank=True, help_text=_("List of available assets to download"), verbose_name=_("Available Assets")) - console_output = models.TextField(null=False, default="", blank=True, help_text=_("Console output of the processing node"), verbose_name=_("Console Output")) orthophoto_extent = GeometryField(null=True, blank=True, srid=4326, help_text=_("Extent of the orthophoto"), verbose_name=_("Orthophoto Extent")) dsm_extent = GeometryField(null=True, blank=True, srid=4326, help_text="Extent of the DSM", verbose_name=_("DSM Extent")) @@ -290,6 +290,8 @@ def __init__(self, *args, **kwargs): # To help keep track of changes to the project id self.__original_project_id = self.project.id + + self.console = Console(self.data_path("console_output.txt")) def __str__(self): name = self.name if self.name is not None else gettext("unnamed") @@ -354,6 +356,12 @@ def assets_path(self, *args): """ return self.task_path("assets", *args) + def data_path(self, *args): + """ + Path to task data that does not fit in database fields (e.g. console output) + """ + return self.task_path("data", *args) + def task_path(self, *args): """ Get path relative to the root task directory @@ -490,7 +498,7 @@ def get_asset_download_path(self, asset): raise FileNotFoundError("{} is not a valid asset".format(asset)) def handle_import(self): - self.console_output += gettext("Importing assets...") + "\n" + self.console += gettext("Importing assets...") + "\n" self.save() zip_path = self.assets_path("all.zip") @@ -709,7 +717,7 @@ def callback(progress): self.options = list(filter(lambda d: d['name'] != 'rerun-from', self.options)) self.upload_progress = 0 - self.console_output = "" + self.console.reset() self.processing_time = -1 self.status = None self.last_error = None @@ -740,10 +748,10 @@ def callback(progress): # Need to update status (first time, queued or running?) if self.uuid and self.status in [None, status_codes.QUEUED, status_codes.RUNNING]: # Update task info from processing node - if not self.console_output: + if not self.console.output(): current_lines_count = 0 else: - current_lines_count = len(self.console_output.split("\n")) + current_lines_count = len(self.console.output().split("\n")) info = self.processing_node.get_task_info(self.uuid, current_lines_count) @@ -751,7 +759,7 @@ def callback(progress): self.status = info.status.value if len(info.output) > 0: - self.console_output += "\n".join(info.output) + '\n' + self.console += "\n".join(info.output) + '\n' # Update running progress self.running_progress = (info.progress / 100.0) * self.TASK_PROGRESS_LAST_VALUE @@ -891,7 +899,7 @@ def extract_assets_and_complete(self): self.update_size() self.potree_scene = {} self.running_progress = 1.0 - self.console_output += gettext("Done!") + "\n" + self.console += gettext("Done!") + "\n" self.status = status_codes.COMPLETED self.save() diff --git a/app/tests/test_api.py b/app/tests/test_api.py index fd392a35f..c97916836 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -140,12 +140,12 @@ def test_projects_and_tasks(self): self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertTrue(res.data == "") - task.console_output = "line1\nline2\nline3" + task.console.reset("line1\nline2\nline3") task.save() res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id)) self.assertEqual(res.status_code, status.HTTP_200_OK) - self.assertTrue(res.data == task.console_output) + self.assertTrue(res.data == task.console.output()) # Console output with line num res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id)) @@ -155,7 +155,7 @@ def test_projects_and_tasks(self): res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id)) self.assertTrue(res.data == "") res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id)) - self.assertTrue(res.data == task.console_output) + self.assertTrue(res.data == task.console.output()) # Cannot list task details for a task belonging to a project we don't have access to res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id)) diff --git a/coreplugins/cloudimport/api_views.py b/coreplugins/cloudimport/api_views.py index a4100dc01..469691040 100644 --- a/coreplugins/cloudimport/api_views.py +++ b/coreplugins/cloudimport/api_views.py @@ -41,7 +41,7 @@ def post(self, request, project_pk=None, pk=None): files = platform.import_from_folder(folder_url) # Update the task with the new information - task.console_output += "Importing {} images...\n".format(len(files)) + task.console += "Importing {} images...\n".format(len(files)) task.images_count = len(files) task.pending_action = pending_actions.IMPORT task.save() diff --git a/coreplugins/dronedb/api_views.py b/coreplugins/dronedb/api_views.py index 1b7bec6b4..673556ad3 100644 --- a/coreplugins/dronedb/api_views.py +++ b/coreplugins/dronedb/api_views.py @@ -181,7 +181,7 @@ def post(self, request, project_pk=None, pk=None): return Response({'error': 'Empty dataset or folder.'}, status=status.HTTP_400_BAD_REQUEST) # Update the task with the new information - task.console_output += "Importing {} images...\n".format(len(files)) + task.console += "Importing {} images...\n".format(len(files)) task.images_count = len(files) task.pending_action = pending_actions.IMPORT task.save() diff --git a/coreplugins/tasknotification/signals.py b/coreplugins/tasknotification/signals.py index ad49e4c11..182367758 100644 --- a/coreplugins/tasknotification/signals.py +++ b/coreplugins/tasknotification/signals.py @@ -22,7 +22,7 @@ def handle_task_completed(sender, task_id, **kwargs): setting = Setting.objects.first() notification_app_name = config_data['notification_app_name'] or settings.app_name - console_output = reverse_output(task.console_output) + console_output = reverse_output(task.console.output()) notification.send( f"{notification_app_name} - {task.project.name} Task Completed", f"{task.project.name}\n{task.name} Completed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}", @@ -41,7 +41,7 @@ def handle_task_removed(sender, task_id, **kwargs): task = Task.objects.get(id=task_id) setting = Setting.objects.first() notification_app_name = config_data['notification_app_name'] or settings.app_name - console_output = reverse_output(task.console_output) + console_output = reverse_output(task.console.output()) notification.send( f"{notification_app_name} - {task.project.name} Task removed", f"{task.project.name}\n{task.name} was removed\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}", @@ -60,7 +60,7 @@ def handle_task_failed(sender, task_id, **kwargs): task = Task.objects.get(id=task_id) setting = Setting.objects.first() notification_app_name = config_data['notification_app_name'] or settings.app_name - console_output = reverse_output(task.console_output) + console_output = reverse_output(task.console.output()) notification.send( f"{notification_app_name} - {task.project.name} Task Failed", f"{task.project.name}\n{task.name} Failed with error: {task.last_error}\nProcessing time:{hours_minutes_secs(task.processing_time)}\n\nConsole Output:{console_output}", diff --git a/package.json b/package.json index 3887114cc..7019d92bd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.0", + "version": "2.1.1", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { From ba2d42b3e5aed0de3241b5c0587993207610a563 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 11 Sep 2023 17:05:03 -0400 Subject: [PATCH 35/95] Fix test --- app/tests/test_api.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/app/tests/test_api.py b/app/tests/test_api.py index c97916836..ba67721e5 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -1,4 +1,5 @@ import datetime +import os from django.contrib.auth.models import User from guardian.shortcuts import assign_perm, get_objects_for_user @@ -140,22 +141,26 @@ def test_projects_and_tasks(self): self.assertEqual(res.status_code, status.HTTP_200_OK) self.assertTrue(res.data == "") + data_path = task.data_path() + if not os.path.exists(data_path): + os.makedirs(data_path, exist_ok=True) + task.console.reset("line1\nline2\nline3") task.save() res = client.get('/api/projects/{}/tasks/{}/output/'.format(project.id, task.id)) self.assertEqual(res.status_code, status.HTTP_200_OK) - self.assertTrue(res.data == task.console.output()) + self.assertEqual(res.data, task.console.output()) # Console output with line num res = client.get('/api/projects/{}/tasks/{}/output/?line=2'.format(project.id, task.id)) - self.assertTrue(res.data == "line3") + self.assertEqual(res.data, "line3") # Console output with line num out of bounds res = client.get('/api/projects/{}/tasks/{}/output/?line=3'.format(project.id, task.id)) - self.assertTrue(res.data == "") + self.assertEqual(res.data, "") res = client.get('/api/projects/{}/tasks/{}/output/?line=-1'.format(project.id, task.id)) - self.assertTrue(res.data == task.console.output()) + self.assertEqual(res.data, task.console.output()) # Cannot list task details for a task belonging to a project we don't have access to res = client.get('/api/projects/{}/tasks/{}/'.format(other_project.id, other_task.id)) From df245905c5fe9a2d3f9f0b86e633004f750bf2e1 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 11 Sep 2023 17:28:47 -0400 Subject: [PATCH 36/95] Safer console writes --- app/classes/console.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/app/classes/console.py b/app/classes/console.py index d9f0a1e25..96e55ed0a 100644 --- a/app/classes/console.py +++ b/app/classes/console.py @@ -31,18 +31,23 @@ def output(self): def append(self, text): if os.path.isdir(self.parent_dir): - # Write - if not os.path.isdir(self.base_dir): - os.makedirs(self.base_dir, exist_ok=True) + try: + # Write + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir, exist_ok=True) + + with open(self.file, "a") as f: + f.write(text) + except IOError: + logger.warn("Cannot append to console file: %s" % self.file) - with open(self.file, "a") as f: - f.write(text) - def reset(self, text = ""): if os.path.isdir(self.parent_dir): - if not os.path.isdir(self.base_dir): - os.makedirs(self.base_dir, exist_ok=True) - - with open(self.file, "w") as f: - f.write(text) - + try: + if not os.path.isdir(self.base_dir): + os.makedirs(self.base_dir, exist_ok=True) + + with open(self.file, "w") as f: + f.write(text) + except IOError: + logger.warn("Cannot reset console file: %s" % self.file) From b7178c830a077b6087fd0d9f07efee98b4346c02 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 12 Sep 2023 11:36:52 -0400 Subject: [PATCH 37/95] Warn when quota is zero --- app/templates/app/dashboard.html | 5 ++--- app/templates/app/quota.html | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/templates/app/dashboard.html b/app/templates/app/dashboard.html index 344981ad2..d2292bf8e 100644 --- a/app/templates/app/dashboard.html +++ b/app/templates/app/dashboard.html @@ -25,7 +25,8 @@

    {% trans 'Welcome!' %} ☺

    {% else %} - + {% include "quota.html" %} + {% if no_tasks %}

    {% trans 'Welcome!' %} ☺

    {% trans 'Select Images and GCP' as upload_images %} @@ -42,8 +43,6 @@

    {% trans 'Welcome!' %} ☺

    {% endif %} - {% include "quota.html" %} -
    {% endif %} diff --git a/app/templates/app/quota.html b/app/templates/app/quota.html index c52df023a..fa80119ce 100644 --- a/app/templates/app/quota.html +++ b/app/templates/app/quota.html @@ -1,11 +1,16 @@ {% load i18n %} {% load settings %} +{% quota_exceeded_grace_period as when %} + {% if user.profile.has_exceeded_quota_cached %} {% with total=user.profile.quota|disk_size used=user.profile.used_quota_cached|disk_size %} - {% quota_exceeded_grace_period as when %}
    {% blocktrans %}The disk quota is being exceeded ({{ used }} of {{ total }} used). The most recent tasks will be automatically deleted {{ when }}, until usage falls below {{ total }}.{% endblocktrans %}
    {% endwith %} +{% elif user.profile.quota == 0 %} +
    + {% blocktrans %}Your account does not have a storage quota. Any new task will be automatically deleted {{ when }}{% endblocktrans %} +
    {% endif %} \ No newline at end of file From b1fd36da26c39621af2bf8ef295cd9c715e48c0a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 12 Sep 2023 11:38:33 -0400 Subject: [PATCH 38/95] Update locale --- .../app/js/translations/odm_autogenerated.js | 148 +++++++++--------- locale | 2 +- 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/app/static/app/js/translations/odm_autogenerated.js b/app/static/app/js/translations/odm_autogenerated.js index d4f81208f..811754bf3 100644 --- a/app/static/app/js/translations/odm_autogenerated.js +++ b/app/static/app/js/translations/odm_autogenerated.js @@ -1,94 +1,94 @@ // Auto-generated with extract_odm_strings.py, do not edit! -_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); -_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); -_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); +_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); +_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); +_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); +_("Permanently delete all previous results and rerun the processing pipeline."); +_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); +_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); _("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); -_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); +_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); +_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Copy output results to this folder after processing."); +_("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); +_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); +_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); +_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); +_("The maximum vertex count of the output mesh. Default: %(default)s"); +_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); +_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); +_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); +_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); +_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); _("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); -_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); +_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); +_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); +_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); _("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); -_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); -_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); +_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); +_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); +_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); _("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); -_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); _("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); -_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); -_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); -_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); -_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); -_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); -_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); +_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); +_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); +_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); +_("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); +_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Displays version number and exits. "); _("Generate OGC 3D Tiles outputs. Default: %(default)s"); -_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); -_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); +_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); +_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); _("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); -_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); -_("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); -_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); -_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); -_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); +_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); +_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); +_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); _("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); -_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); -_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); _("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); -_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); -_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); -_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); _("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); -_("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); +_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); +_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); +_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); +_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); _("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); -_("The maximum vertex count of the output mesh. Default: %(default)s"); -_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); -_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); -_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); -_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); -_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); +_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); +_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); +_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); _("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); -_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); -_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); -_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); -_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); -_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); -_("show this help message and exit"); -_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); +_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); +_("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); +_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); +_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); +_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); _("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); -_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); -_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); -_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); -_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); -_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); -_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); -_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); -_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); -_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); -_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); _("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); -_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); -_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); -_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); +_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); +_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); +_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); _("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); -_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); -_("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); -_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); -_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); +_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); +_("Skip the blending of colors near seams. Default: %(default)s"); +_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); +_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); +_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); +_("show this help message and exit"); +_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); +_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); _("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); -_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); -_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); _("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); -_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); -_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); -_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); -_("Displays version number and exits. "); -_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); -_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); -_("Permanently delete all previous results and rerun the processing pipeline."); -_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); -_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); -_("Copy output results to this folder after processing."); -_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); +_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); _("Export the georeferenced point cloud in CSV format. Default: %(default)s"); -_("Skip the blending of colors near seams. Default: %(default)s"); +_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); diff --git a/locale b/locale index 6469d33dc..5776dd59a 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 6469d33dccdc2b7cc4c3596e9f11dfc907736e28 +Subproject commit 5776dd59a1442696eb227d7d93a26b16c95c1276 From 8059900a58b57599221b913ad355a932a3b2e602 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 12 Sep 2023 12:40:45 -0400 Subject: [PATCH 39/95] task_count check in quota removal --- worker/tasks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/worker/tasks.py b/worker/tasks.py index 3cafcdbd4..ca290e132 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -217,6 +217,8 @@ def check_quotas(): if now > deadline: # deadline passed, delete tasks until quota is met logger.info("Quota deadline expired for %s, deleting tasks" % str(p.user.username)) + task_count = Task.objects.filter(project__owner=p.user).count() + c = 0 while p.has_exceeded_quota(): try: @@ -227,6 +229,9 @@ def check_quotas(): last_task.delete() except Exception as e: logger.warn("Cannot delete %s for %s: %s" % (str(last_task), str(p.user.username), str(e))) + + c += 1 + if c >= task_count: break else: p.clear_quota_deadline() From 14aad55245f0cb47ebc609f8cc3d26814ae178b3 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 12 Sep 2023 18:06:52 -0400 Subject: [PATCH 40/95] Add test --- app/tests/test_external_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/tests/test_external_auth.py b/app/tests/test_external_auth.py index 2928bebd7..1056bf17b 100644 --- a/app/tests/test_external_auth.py +++ b/app/tests/test_external_auth.py @@ -46,3 +46,6 @@ def test_ext_auth(self): # Re-test login ok = client.login(username='extuser1', password='test1234') self.assertTrue(ok) + + # Check that the user has been added to the default group + self.assertTrue(user.groups.filter(name='Default').exists()) From 35418824237129385164131916ca864c929eea19 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 12 Sep 2023 18:34:56 -0400 Subject: [PATCH 41/95] Return 100 when den is zero --- app/templatetags/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templatetags/settings.py b/app/templatetags/settings.py index b0962cbd2..d69d378ef 100644 --- a/app/templatetags/settings.py +++ b/app/templatetags/settings.py @@ -32,7 +32,7 @@ def disk_size(megabytes): @register.simple_tag def percentage(num, den, maximum=None): if den == 0: - return 0 + return 100 perc = max(0, num / den * 100) if maximum is not None: perc = min(perc, maximum) From 95085301c2ef1a5bf43196fa65e587113ab2d445 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 14 Sep 2023 23:32:09 -0400 Subject: [PATCH 42/95] Update presets --- app/boot.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/app/boot.py b/app/boot.py index f40f48036..d95b7da11 100644 --- a/app/boot.py +++ b/app/boot.py @@ -110,8 +110,7 @@ def add_default_presets(): defaults={'options': [{'name': 'auto-boundary', 'value': True}, {'name': 'dsm', 'value': True}, {'name': 'dem-resolution', 'value': '2'}, - {'name': 'pc-quality', 'value': 'high'}, - {'name': 'use-3dmesh', 'value': True}]}) + {'name': 'pc-quality', 'value': 'high'}]}) Preset.objects.update_or_create(name='3D Model', system=True, defaults={'options': [{'name': 'auto-boundary', 'value': True}, {'name': 'mesh-octree-depth', 'value': "12"}, @@ -121,19 +120,8 @@ def add_default_presets(): Preset.objects.update_or_create(name='Buildings', system=True, defaults={'options': [{'name': 'auto-boundary', 'value': True}, {'name': 'mesh-size', 'value': '300000'}, - {'name': 'pc-geometric', 'value': True}, {'name': 'feature-quality', 'value': 'high'}, {'name': 'pc-quality', 'value': 'high'}]}) - Preset.objects.update_or_create(name='Buildings Ultra Quality', system=True, - defaults={'options': [{'name': 'auto-boundary', 'value': True}, - {'name': 'mesh-size', 'value': '300000'}, - {'name': 'pc-geometric', 'value': True}, - {'name': 'feature-quality', 'value': 'ultra'}, - {'name': 'pc-quality', 'value': 'ultra'}]}) - Preset.objects.update_or_create(name='Point of Interest', system=True, - defaults={'options': [{'name': 'auto-boundary', 'value': True}, - {'name': 'mesh-size', 'value': '300000'}, - {'name': 'use-3dmesh', 'value': True}]}) Preset.objects.update_or_create(name='Forest', system=True, defaults={'options': [{'name': 'auto-boundary', 'value': True}, {'name': 'min-num-features', 'value': '18000'}, From be082a7d717c2f11a4c4eacc7bdded749ebf7390 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 13:11:48 -0400 Subject: [PATCH 43/95] Check for maxImages on frontend --- app/static/app/js/components/EditTaskForm.jsx | 15 +- app/static/app/js/components/NewTaskPanel.jsx | 15 +- .../app/js/components/ProjectListItem.jsx | 6 +- app/static/app/js/css/NewTaskPanel.scss | 5 + .../app/js/translations/odm_autogenerated.js | 152 +++++++++--------- locale | 2 +- 6 files changed, 114 insertions(+), 81 deletions(-) diff --git a/app/static/app/js/components/EditTaskForm.jsx b/app/static/app/js/components/EditTaskForm.jsx index a4330ea12..05fb7abab 100644 --- a/app/static/app/js/components/EditTaskForm.jsx +++ b/app/static/app/js/components/EditTaskForm.jsx @@ -85,6 +85,18 @@ class EditTaskForm extends React.Component { this.state.selectedPreset; } + checkFilesCount(filesCount){ + if (!this.state.selectedNode) return false; + if (filesCount === 0) return true; + if (this.state.selectedNode.max_images === null) return true; + return this.state.selectedNode.max_images >= filesCount; + } + + selectedNodeMaxImages(){ + if (!this.state.selectedNode) return null; + return this.state.selectedNode.max_images; + } + notifyFormLoaded(){ if (this.props.onFormLoaded && this.formReady()) this.props.onFormLoaded(); } @@ -115,8 +127,6 @@ class EditTaskForm extends React.Component { return; } - let now = new Date(); - let nodes = json.map(node => { return { id: node.id, @@ -124,6 +134,7 @@ class EditTaskForm extends React.Component { label: `${node.label} (queue: ${node.queue_count})`, options: node.available_options, queue_count: node.queue_count, + max_images: node.max_images, enabled: node.online, url: `http://${node.hostname}:${node.port}` }; diff --git a/app/static/app/js/components/NewTaskPanel.jsx b/app/static/app/js/components/NewTaskPanel.jsx index 2bc28218b..f72a2256a 100644 --- a/app/static/app/js/components/NewTaskPanel.jsx +++ b/app/static/app/js/components/NewTaskPanel.jsx @@ -124,11 +124,24 @@ class NewTaskPanel extends React.Component { } render() { + let filesCountOk = true; + if (this.taskForm && !this.taskForm.checkFilesCount(this.props.filesCount)) filesCountOk = false; + return (

    {interpolate(_("%(count)s files selected. Please check these additional options:"), { count: this.props.filesCount})}

    + + {!filesCountOk ? +
    + {interpolate(_("Number of files selected exceeds the maximum of %(count)s allowed on this processing node."), { count: this.taskForm.selectedNodeMaxImages() })} + +
    + : ""} + {_("Loading…")} : - + }
    diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 31d6ec429..cab4a54da 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -281,7 +281,7 @@ class ProjectListItem extends React.Component { }); } }) - .on("reset", () => { + .on("reset", (e) => { this.resetUploadState(); }) .on("dragenter", () => { @@ -398,6 +398,10 @@ class ProjectListItem extends React.Component { this.resetUploadState(); } + handleClearFiles = () => { + this.dz.removeAllFiles(true); + } + handleUpload = () => { // Not a second click for adding more files? if (!this.state.upload.editing){ diff --git a/app/static/app/js/css/NewTaskPanel.scss b/app/static/app/js/css/NewTaskPanel.scss index 947bbeca5..84d0d2d1d 100644 --- a/app/static/app/js/css/NewTaskPanel.scss +++ b/app/static/app/js/css/NewTaskPanel.scss @@ -41,4 +41,9 @@ opacity: 0.8; pointer-events:none; } + + button.redo{ + margin-top: 0; + margin-left: 10px; + } } \ No newline at end of file diff --git a/app/static/app/js/translations/odm_autogenerated.js b/app/static/app/js/translations/odm_autogenerated.js index 811754bf3..2a5306cb2 100644 --- a/app/static/app/js/translations/odm_autogenerated.js +++ b/app/static/app/js/translations/odm_autogenerated.js @@ -1,94 +1,94 @@ // Auto-generated with extract_odm_strings.py, do not edit! -_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); +_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); +_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); +_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); +_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); +_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); _("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); -_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); -_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); -_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); -_("Permanently delete all previous results and rerun the processing pipeline."); -_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); -_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); -_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); -_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); +_("show this help message and exit"); +_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); +_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); +_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); +_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); +_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); +_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); _("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); -_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); +_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); +_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); _("Copy output results to this folder after processing."); -_("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); -_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); -_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); -_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); +_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); +_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); +_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); +_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); +_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); +_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); +_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); _("The maximum vertex count of the output mesh. Default: %(default)s"); -_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); -_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); -_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); +_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); _("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); -_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); +_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); +_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); +_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); +_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); +_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); _("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); -_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); -_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); -_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); -_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); -_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); -_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); -_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); -_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); -_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); -_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); -_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); -_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); +_("Permanently delete all previous results and rerun the processing pipeline."); +_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); _("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); -_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); -_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); -_("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); -_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); -_("Displays version number and exits. "); +_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); +_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); +_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); +_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); +_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); +_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); +_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); _("Generate OGC 3D Tiles outputs. Default: %(default)s"); -_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); -_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); -_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); -_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); -_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); -_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); _("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); -_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); +_("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); +_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); +_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); +_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); +_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); +_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); +_("Displays version number and exits. "); _("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); -_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); -_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); -_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); -_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); +_("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); +_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); _("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); -_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); -_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); +_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); +_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); +_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); _("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); -_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); -_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); -_("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); -_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); -_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); +_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); +_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); +_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); _("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); -_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); -_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); -_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); -_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); +_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); _("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); -_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); -_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); -_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); -_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); -_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); -_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); -_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); -_("Skip the blending of colors near seams. Default: %(default)s"); -_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); +_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); +_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); _("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); -_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); -_("show this help message and exit"); -_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); -_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); -_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); -_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); -_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); +_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); +_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); _("Export the georeferenced point cloud in CSV format. Default: %(default)s"); -_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); +_("Skip the blending of colors near seams. Default: %(default)s"); +_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); +_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); diff --git a/locale b/locale index 5776dd59a..dbe1eb130 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 5776dd59a1442696eb227d7d93a26b16c95c1276 +Subproject commit dbe1eb130a4efeda1cd67eeb60736a4c647dad63 From 3254968b637becb2f6feab91893e6ac830367d6c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 13:13:54 -0400 Subject: [PATCH 44/95] Cleanup --- app/static/app/js/components/ProjectListItem.jsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index cab4a54da..31d6ec429 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -281,7 +281,7 @@ class ProjectListItem extends React.Component { }); } }) - .on("reset", (e) => { + .on("reset", () => { this.resetUploadState(); }) .on("dragenter", () => { @@ -398,10 +398,6 @@ class ProjectListItem extends React.Component { this.resetUploadState(); } - handleClearFiles = () => { - this.dz.removeAllFiles(true); - } - handleUpload = () => { // Not a second click for adding more files? if (!this.state.upload.editing){ From 74dc45a8cadb7a6df6680937fd142f26b72e7e97 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 13:22:02 -0400 Subject: [PATCH 45/95] Add liveupdate command --- package.json | 2 +- webodm.sh | 71 +++++++++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/package.json b/package.json index 7019d92bd..70411580f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.1", + "version": "2.1.2", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { diff --git a/webodm.sh b/webodm.sh index 5146101ce..5320cab40 100755 --- a/webodm.sh +++ b/webodm.sh @@ -154,6 +154,7 @@ usage(){ echo " stop Stop WebODM" echo " down Stop and remove WebODM's docker containers" echo " update Update WebODM to the latest release" + echo " liveupdate Update WebODM to the latest release without stopping it" echo " rebuild Rebuild all docker containers and perform cleanups" echo " checkenv Do an environment check and install missing components" echo " test Run the unit test suite (developers only)" @@ -493,6 +494,40 @@ resetpassword(){ fi } +update(){ + echo "Updating WebODM..." + + hash git 2>/dev/null || git_not_found=true + if [[ $git_not_found ]]; then + echo "Skipping source update (git not found)" + else + if [[ -d .git ]]; then + run "git pull origin master" + else + echo "Skipping source update (.git directory not found)" + fi + fi + + command="$docker_compose -f docker-compose.yml" + + if [[ $WO_DEFAULT_NODES -gt 0 ]]; then + if [ "${GPU_NVIDIA}" = true ]; then + command+=" -f docker-compose.nodeodm.gpu.nvidia.yml" + elif [ "${GPU_INTEL}" = true ]; then + command+=" -f docker-compose.nodeodm.gpu.intel.yml" + else + command+=" -f docker-compose.nodeodm.yml" + fi + fi + + if [[ $load_micmac_node = true ]]; then + command+=" -f docker-compose.nodemicmac.yml" + fi + + command+=" pull" + run "$command" +} + if [[ $1 = "start" ]]; then environment_check start @@ -528,38 +563,12 @@ elif [[ $1 = "rebuild" ]]; then elif [[ $1 = "update" ]]; then environment_check down - echo "Updating WebODM..." - - hash git 2>/dev/null || git_not_found=true - if [[ $git_not_found ]]; then - echo "Skipping source update (git not found)" - else - if [[ -d .git ]]; then - run "git pull origin master" - else - echo "Skipping source update (.git directory not found)" - fi - fi - - command="$docker_compose -f docker-compose.yml" - - if [[ $WO_DEFAULT_NODES -gt 0 ]]; then - if [ "${GPU_NVIDIA}" = true ]; then - command+=" -f docker-compose.nodeodm.gpu.nvidia.yml" - elif [ "${GPU_INTEL}" = true ]; then - command+=" -f docker-compose.nodeodm.gpu.intel.yml" - else - command+=" -f docker-compose.nodeodm.yml" - fi - fi - - if [[ $load_micmac_node = true ]]; then - command+=" -f docker-compose.nodemicmac.yml" - fi - - command+=" pull" - run "$command" + update echo -e "\033[1mDone!\033[0m You can now start WebODM by running $0 start" +elif [[ $1 = "liveupdate" ]]; then + environment_check + update + echo -e "\033[1mDone!\033[0m You can now finish the update by running $0 restart" elif [[ $1 = "checkenv" ]]; then environment_check elif [[ $1 = "test" ]]; then From 9bfdf9c32077d190725f5dbd29fef50d1396334d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 13:49:12 -0400 Subject: [PATCH 46/95] Fix tests --- app/tests/test_api_preset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/tests/test_api_preset.py b/app/tests/test_api_preset.py index f51d29dd4..2a8fdc3d4 100644 --- a/app/tests/test_api_preset.py +++ b/app/tests/test_api_preset.py @@ -27,7 +27,6 @@ def check_default_presets(self): self.assertTrue(Preset.objects.filter(name="Forest", system=True).exists()) self.assertTrue(Preset.objects.filter(name="Buildings", system=True).exists()) self.assertTrue(Preset.objects.filter(name="3D Model", system=True).exists()) - self.assertTrue(Preset.objects.filter(name="Point of Interest", system=True).exists()) self.assertTrue(Preset.objects.filter(name="Multispectral", system=True).exists()) def test_preset(self): @@ -58,7 +57,7 @@ def test_preset(self): self.assertTrue(res.status_code == status.HTTP_200_OK) # Only ours and global presets are available - self.assertEqual(len(res.data), 15) + self.assertTrue(len(res.data) > 0) self.assertTrue('My Local Preset' in [preset['name'] for preset in res.data]) self.assertTrue('High Resolution' in [preset['name'] for preset in res.data]) self.assertTrue('Global Preset #1' in [preset['name'] for preset in res.data]) From 93704420c6efe70c2638d0cd8f016701fd620f00 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 14:11:23 -0400 Subject: [PATCH 47/95] Add --worker-memory parameter --- docker-compose.worker-memory.yml | 4 ++++ webodm.sh | 13 +++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 docker-compose.worker-memory.yml diff --git a/docker-compose.worker-memory.yml b/docker-compose.worker-memory.yml new file mode 100644 index 000000000..b2f7b98a0 --- /dev/null +++ b/docker-compose.worker-memory.yml @@ -0,0 +1,4 @@ +version: '2.1' +services: + worker: + mem_limit: ${WO_WORKER_MEMORY} \ No newline at end of file diff --git a/webodm.sh b/webodm.sh index 5320cab40..6e4bb3959 100755 --- a/webodm.sh +++ b/webodm.sh @@ -135,6 +135,12 @@ case $key in export WO_SETTINGS shift # past argument shift # past value + ;; + --worker-memory) + WO_WORKER_MEMORY="$2" + export WO_WORKER_MEMORY + shift # past argument + shift # past value ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later @@ -178,6 +184,8 @@ usage(){ echo " --detached Run WebODM in detached mode. This means WebODM will run in the background, without blocking the terminal (default: disabled)" echo " --gpu Use GPU NodeODM nodes (Linux only) (default: disabled)" echo " --settings Path to a settings.py file to enable modifications of system settings (default: None)" + echo " --worker-memory Maximum amount of memory allocated for the worker process (default: unlimited)" + exit } @@ -348,6 +356,7 @@ start(){ echo "Celery Broker: $WO_BROKER" echo "Default Nodes: $WO_DEFAULT_NODES" echo "Settings: $WO_SETTINGS" + echo "Worker memory limit: $WO_WORKER_MEMORY" echo "================================" echo "Make sure to issue a $0 down if you decide to change the environment." echo "" @@ -418,6 +427,10 @@ start(){ command+=" -f docker-compose.settings.yml" fi + if [ ! -z "$WO_WORKER_MEMORY" ]; then + command+=" -f docker-compose.worker-memory.yml" + fi + command="$command up" if [[ $detached = true ]]; then From 9f5c58fe9af3a3485d067ab3912decb70ed83a1d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 16:33:14 -0400 Subject: [PATCH 48/95] Fix migration on Windows --- app/migrations/0038_remove_task_console_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/migrations/0038_remove_task_console_output.py b/app/migrations/0038_remove_task_console_output.py index 88fc024f2..38a54625c 100644 --- a/app/migrations/0038_remove_task_console_output.py +++ b/app/migrations/0038_remove_task_console_output.py @@ -21,7 +21,7 @@ def dump_console_outputs(apps, schema_editor): os.makedirs(dp, exist_ok=True) outfile = os.path.join(dp, "console_output.txt") - with open(outfile, "w") as f: + with open(outfile, "w", encoding="utf-8") as f: f.write(t.console_output) print("Wrote console output for %s to %s" % (t, outfile)) else: From c54857d6e9ffc192e9f782abcf351a96fad1bbea Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 15 Sep 2023 16:47:24 -0400 Subject: [PATCH 49/95] Do not boot on flush --- app/boot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/boot.py b/app/boot.py index d95b7da11..6ccedd12a 100644 --- a/app/boot.py +++ b/app/boot.py @@ -26,7 +26,7 @@ def boot(): # booted is a shared memory variable to keep track of boot status # as multiple gunicorn workers could trigger the boot sequence twice - if (not settings.DEBUG and booted.value) or settings.MIGRATING: return + if (not settings.DEBUG and booted.value) or settings.MIGRATING or settings.FLUSHING: return booted.value = True logger = logging.getLogger('app.logger') From c6d4c763f0eae75914ee5b1d55754b5c2830f4ac Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 16 Sep 2023 10:55:04 -0400 Subject: [PATCH 50/95] Add UI_MAX_PROCESSING_NODES setting --- app/templatetags/processingnode_extras.py | 7 ++++++- webodm/settings.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/templatetags/processingnode_extras.py b/app/templatetags/processingnode_extras.py index 1abb77b16..8aa289837 100644 --- a/app/templatetags/processingnode_extras.py +++ b/app/templatetags/processingnode_extras.py @@ -1,5 +1,6 @@ from django import template from guardian.shortcuts import get_objects_for_user +from webodm import settings from nodeodm.models import ProcessingNode @@ -8,7 +9,11 @@ @register.simple_tag(takes_context=True) def get_visible_processing_nodes(context): - return get_objects_for_user(context['request'].user, "nodeodm.view_processingnode", ProcessingNode, accept_global_perms=False) + queryset = get_objects_for_user(context['request'].user, "nodeodm.view_processingnode", ProcessingNode, accept_global_perms=False) + if settings.UI_MAX_PROCESSING_NODES is not None: + return queryset[:settings.UI_MAX_PROCESSING_NODES] + else: + return queryset @register.simple_tag(takes_context=True) diff --git a/webodm/settings.py b/webodm/settings.py index 04389728e..17fcb4865 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -398,6 +398,8 @@ def scalebyiv(color, n): # from an account that is exceeding a disk quota QUOTA_EXCEEDED_GRACE_PERIOD = 8 +# Maximum number of processing nodes to show in the "Processing Nodes" menu +UI_MAX_PROCESSING_NODES = None if TESTING or FLUSHING: CELERY_TASK_ALWAYS_EAGER = True From 82f3408b9412f2b1b408f3b19962018cf8e8efec Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 16 Sep 2023 11:26:24 -0400 Subject: [PATCH 51/95] Add test, comments --- app/tests/test_app.py | 19 +++++++++++++++++++ app/views/app.py | 2 +- webodm/settings.py | 3 +++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/app/tests/test_app.py b/app/tests/test_app.py index a0d9560a9..d626dd85b 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -1,6 +1,8 @@ from django.contrib.auth.models import User, Group from django.test import Client from rest_framework import status +from guardian.shortcuts import assign_perm +from nodeodm.models import ProcessingNode from app.models import Project, Task from app.models import Setting @@ -24,6 +26,11 @@ def setUp(self): # Add user to test Group User.objects.get(pk=1).groups.add(my_group) + # Add view permissions + user = User.objects.get(username=self.credentials['username']) + pns = ProcessingNode.objects.all() + for pn in pns: + assign_perm('view_processingnode', user, pn) def test_user_login(self): c = Client() @@ -89,6 +96,18 @@ def test_views(self): self.assertEqual(message.tags, 'warning') self.assertTrue("offline" in message.message) + # The menu should have 3 processing nodes + res = c.get('/dashboard/', follow=True) + self.assertEqual(res.content.decode("utf-8").count('href="/processingnode/'), 3) + self.assertTemplateUsed(res, 'app/dashboard.html') + + # We can change that with a setting + settings.UI_MAX_PROCESSING_NODES = 1 + + res = c.get('/dashboard/', follow=True) + self.assertEqual(res.content.decode("utf-8").count('href="/processingnode/'), 1) + self.assertTemplateUsed(res, 'app/dashboard.html') + res = c.get('/processingnode/9999/') self.assertTrue(res.status_code == 404) diff --git a/app/views/app.py b/app/views/app.py index f37266e24..85fd2a013 100644 --- a/app/views/app.py +++ b/app/views/app.py @@ -109,7 +109,7 @@ def about(request): def processing_node(request, processing_node_id): pn = get_object_or_404(ProcessingNode, pk=processing_node_id) if not pn.update_node_info(): - messages.add_message(request, messages.constants.WARNING, '{} seems to be offline.'.format(pn)) + messages.add_message(request, messages.constants.WARNING, _('%(node)s seems to be offline.') % {'node': pn}) return render(request, 'app/processing_node.html', { diff --git a/webodm/settings.py b/webodm/settings.py index 17fcb4865..6538d698c 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -391,7 +391,10 @@ def scalebyiv(color, n): # before it should be considered offline NODE_OFFLINE_MINUTES = 5 +# URL to external auth endpoint EXTERNAL_AUTH_ENDPOINT = '' + +# URL to a page where a user can reset the password RESET_PASSWORD_LINK = '' # Number of hours before tasks are automatically deleted From c0fe40715744b760b42636a369cd4178c4b7e87a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 16 Sep 2023 12:23:49 -0400 Subject: [PATCH 52/95] Add NODE_OPTIMISTIC_MODE --- app/api/processingnodes.py | 14 +++++++++++++- app/static/app/js/components/EditTaskForm.jsx | 2 +- app/tests/test_api.py | 12 ++++++++++++ app/tests/test_app.py | 9 +++++++++ nodeodm/models.py | 3 +++ webodm/settings.py | 8 ++++++-- worker/tasks.py | 3 +++ 7 files changed, 47 insertions(+), 4 deletions(-) diff --git a/app/api/processingnodes.py b/app/api/processingnodes.py index ed37c384e..9195714a5 100644 --- a/app/api/processingnodes.py +++ b/app/api/processingnodes.py @@ -6,7 +6,7 @@ from rest_framework.views import APIView from nodeodm.models import ProcessingNode - +from webodm import settings class ProcessingNodeSerializer(serializers.ModelSerializer): online = serializers.SerializerMethodField() @@ -49,6 +49,18 @@ class ProcessingNodeViewSet(viewsets.ModelViewSet): serializer_class = ProcessingNodeSerializer queryset = ProcessingNode.objects.all() + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + + if settings.UI_MAX_PROCESSING_NODES is not None: + queryset = queryset[:settings.UI_MAX_PROCESSING_NODES] + + if settings.NODE_OPTIMISTIC_MODE: + for pn in queryset: + pn.update_node_info() + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) class ProcessingNodeOptionsView(APIView): """ diff --git a/app/static/app/js/components/EditTaskForm.jsx b/app/static/app/js/components/EditTaskForm.jsx index 05fb7abab..64a7ae179 100644 --- a/app/static/app/js/components/EditTaskForm.jsx +++ b/app/static/app/js/components/EditTaskForm.jsx @@ -86,7 +86,7 @@ class EditTaskForm extends React.Component { } checkFilesCount(filesCount){ - if (!this.state.selectedNode) return false; + if (!this.state.selectedNode) return true; if (filesCount === 0) return true; if (this.state.selectedNode.max_images === null) return true; return this.state.selectedNode.max_images >= filesCount; diff --git a/app/tests/test_api.py b/app/tests/test_api.py index ba67721e5..9e757562a 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -460,6 +460,18 @@ def test_processingnodes(self): self.assertTrue(len(res.data) == 1) self.assertTrue(res.data[0]['name'] == 'a') + # Test optimistic mode + self.assertFalse(p4.is_online()) + + settings.NODE_OPTIMISTIC_MODE = True + + self.assertTrue(p4.is_online()) + res = client.get('/api/processingnodes/') + self.assertEqual(len(res.data), 3) + for nodes in res.data: + self.assertTrue(nodes['online']) + + settings.NODE_OPTIMISTIC_MODE = False def test_token_auth(self): client = APIClient() diff --git a/app/tests/test_app.py b/app/tests/test_app.py index d626dd85b..d0bccd570 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -101,6 +101,10 @@ def test_views(self): self.assertEqual(res.content.decode("utf-8").count('href="/processingnode/'), 3) self.assertTemplateUsed(res, 'app/dashboard.html') + # The API should return 3 nodes + res = c.get('/api/processingnodes/') + self.assertEqual(len(res.data), 3) + # We can change that with a setting settings.UI_MAX_PROCESSING_NODES = 1 @@ -108,6 +112,11 @@ def test_views(self): self.assertEqual(res.content.decode("utf-8").count('href="/processingnode/'), 1) self.assertTemplateUsed(res, 'app/dashboard.html') + res = c.get('/api/processingnodes/') + self.assertEqual(len(res.data), 1) + + settings.UI_MAX_PROCESSING_NODES = None + res = c.get('/processingnode/9999/') self.assertTrue(res.status_code == 404) diff --git a/nodeodm/models.py b/nodeodm/models.py index a2f8e81e3..270abf036 100644 --- a/nodeodm/models.py +++ b/nodeodm/models.py @@ -52,6 +52,9 @@ def find_best_available_node(): .order_by('queue_count').first() def is_online(self): + if settings.NODE_OPTIMISTIC_MODE: + return True + return self.last_refreshed is not None and \ self.last_refreshed >= timezone.now() - timedelta(minutes=settings.NODE_OFFLINE_MINUTES) diff --git a/webodm/settings.py b/webodm/settings.py index 6538d698c..fa7b5006a 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -389,7 +389,11 @@ def scalebyiv(color, n): # Number of minutes a processing node hasn't been seen # before it should be considered offline -NODE_OFFLINE_MINUTES = 5 +NODE_OFFLINE_MINUTES = 5 + +# When turned on, updates nodes information only when necessary +# and assumes that all nodes are always online, avoiding polling +NODE_OPTIMISTIC_MODE = False # URL to external auth endpoint EXTERNAL_AUTH_ENDPOINT = '' @@ -401,7 +405,7 @@ def scalebyiv(color, n): # from an account that is exceeding a disk quota QUOTA_EXCEEDED_GRACE_PERIOD = 8 -# Maximum number of processing nodes to show in the "Processing Nodes" menu +# Maximum number of processing nodes to show in "Processing Nodes" menus/dropdowns UI_MAX_PROCESSING_NODES = None if TESTING or FLUSHING: diff --git a/worker/tasks.py b/worker/tasks.py index ca290e132..12ff4f1e6 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -33,6 +33,9 @@ @app.task def update_nodes_info(): + if settings.NODE_OPTIMISTIC_MODE: + return + processing_nodes = ProcessingNode.objects.all() for processing_node in processing_nodes: processing_node.update_node_info() From ef5336927d50e3f2f9c6fe1dcfc78b24a51879f5 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 16 Sep 2023 13:46:15 -0400 Subject: [PATCH 53/95] Persists secret_key between updates --- .dockerignore | 1 + .gitignore | 3 ++- docker-compose.yml | 2 ++ webodm.sh | 26 ++++++++++++++++++++------ webodm/settings.py | 27 +++++++++++++++------------ 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/.dockerignore b/.dockerignore index f3b64113e..75babffc1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ **/.git +.secret_key \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2e07b9d45..d69cbb8b0 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,5 @@ package-lock.json # Debian builds dpkg/build -dpkg/deb \ No newline at end of file +dpkg/deb +.secret_key \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 23b8922fd..9c460d99b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: - WO_BROKER - WO_DEV - WO_DEV_WATCH_PLUGINS + - WO_SECRET_KEY restart: unless-stopped oom_score_adj: 0 broker: @@ -52,5 +53,6 @@ services: environment: - WO_BROKER - WO_DEBUG + - WO_SECRET_KEY restart: unless-stopped oom_score_adj: 250 diff --git a/webodm.sh b/webodm.sh index 6e4bb3959..0c794bf77 100755 --- a/webodm.sh +++ b/webodm.sh @@ -335,13 +335,27 @@ run(){ eval "$1" } +get_secret(){ + if [ ! -e ./.secret_key ] && [ -e /dev/random ]; then + echo "Generating secret in ./.secret_key" + export WO_SECRET_KEY=$(head -c50 < /dev/random | base64) + echo $WO_SECRET_KEY > ./.secret_key + elif [ -e ./.secret_key ]; then + export WO_SECRET_KEY=$(cat ./.secret_key) + else + export WO_SECRET_KEY="" + fi +} + start(){ - if [[ $dev_mode = true ]]; then - echo "Starting WebODM in development mode..." - down - else - echo "Starting WebODM..." - fi + get_secret + + if [[ $dev_mode = true ]]; then + echo "Starting WebODM in development mode..." + down + else + echo "Starting WebODM..." + fi echo "" echo "Using the following environment:" echo "================================" diff --git a/webodm/settings.py b/webodm/settings.py index fa7b5006a..498067629 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -27,18 +27,21 @@ try: from .secret_key import SECRET_KEY except ImportError: - # This will be executed the first time Django runs - # It generates a secret_key.py file that contains the SECRET_KEY - from django.utils.crypto import get_random_string - - current_dir = os.path.abspath(os.path.dirname(__file__)) - chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' - secret = get_random_string(50, chars) - with open(os.path.join(current_dir, 'secret_key.py'), 'w') as f: - f.write("SECRET_KEY='{}'".format(secret)) - SECRET_KEY=secret - - print("Generated secret key") + if os.environ.get("WO_SECRET_KEY", "") != "": + SECRET_KEY = os.environ.get("WO_SECRET_KEY") + else: + # This will be executed the first time Django runs + # It generates a secret_key.py file that contains the SECRET_KEY + from django.utils.crypto import get_random_string + + current_dir = os.path.abspath(os.path.dirname(__file__)) + chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' + secret = get_random_string(50, chars) + with open(os.path.join(current_dir, 'secret_key.py'), 'w') as f: + f.write("SECRET_KEY='{}'".format(secret)) + SECRET_KEY=secret + + print("Generated secret key") with open(os.path.join(BASE_DIR, 'package.json')) as package_file: data = json.load(package_file) From 950d54d51b8183992e5131bf9f9142dac6c38b1e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 16 Sep 2023 13:49:34 -0400 Subject: [PATCH 54/95] Update locales --- .../app/js/translations/odm_autogenerated.js | 154 +++++++++--------- locale | 2 +- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/app/static/app/js/translations/odm_autogenerated.js b/app/static/app/js/translations/odm_autogenerated.js index 2a5306cb2..bb87910bb 100644 --- a/app/static/app/js/translations/odm_autogenerated.js +++ b/app/static/app/js/translations/odm_autogenerated.js @@ -1,94 +1,94 @@ // Auto-generated with extract_odm_strings.py, do not edit! -_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); -_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); -_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); -_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); -_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); -_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); -_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); -_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); -_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); _("show this help message and exit"); -_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); -_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); -_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); -_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); -_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); -_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); -_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); -_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); -_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); -_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); -_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); -_("Copy output results to this folder after processing."); -_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); -_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); -_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); -_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); -_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); -_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); -_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); -_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); -_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); -_("The maximum vertex count of the output mesh. Default: %(default)s"); -_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); +_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); +_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); +_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); _("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); -_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); -_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); +_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); _("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); -_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); -_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); -_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); -_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); -_("Permanently delete all previous results and rerun the processing pipeline."); -_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); +_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); +_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); _("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); _("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); +_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); +_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); +_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); _("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); -_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); +_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); +_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); +_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); _("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); -_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); -_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); -_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); +_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); +_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); +_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); +_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); +_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); +_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); +_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); +_("Copy output results to this folder after processing."); +_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); +_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); _("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); -_("Generate OGC 3D Tiles outputs. Default: %(default)s"); +_("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); +_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); +_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); +_("The maximum vertex count of the output mesh. Default: %(default)s"); +_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); +_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); +_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); +_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); +_("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); +_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); +_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); +_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); +_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); _("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); +_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); +_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); +_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); _("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); -_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); -_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); -_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); -_("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); +_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); +_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); +_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); +_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Skip the blending of colors near seams. Default: %(default)s"); +_("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); +_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); +_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); +_("Generate OGC 3D Tiles outputs. Default: %(default)s"); +_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); +_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); _("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); +_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); +_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); +_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); +_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); +_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); +_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); +_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); +_("Permanently delete all previous results and rerun the processing pipeline."); +_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); +_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); +_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); _("Displays version number and exits. "); _("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); -_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); -_("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); -_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); -_("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); -_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); -_("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); -_("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); -_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); -_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); -_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); -_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); -_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); -_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); +_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); +_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); _("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); +_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); +_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); _("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); -_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); -_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); -_("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); -_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); -_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); -_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); -_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); -_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); -_("Skip the blending of colors near seams. Default: %(default)s"); -_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); -_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); -_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); +_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); diff --git a/locale b/locale index dbe1eb130..d253dd577 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit dbe1eb130a4efeda1cd67eeb60736a4c647dad63 +Subproject commit d253dd5770c42a0705d0f861db3314b08f230a68 From e2b7de81d3f2a4674eb14f084b1330d5402e3e7e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 14:08:45 -0400 Subject: [PATCH 55/95] Chunked import uploads --- app/api/tasks.py | 59 +++++++++++++++---- .../app/js/components/ImportTaskPanel.jsx | 4 +- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index 191178b6f..26f612c77 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -1,9 +1,11 @@ import os +import re +import shutil from wsgiref.util import FileWrapper import mimetypes -from shutil import copyfileobj +from shutil import copyfileobj, move from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError from django.core.files.uploadedfile import InMemoryUploadedFile from django.db import transaction @@ -23,7 +25,7 @@ from .tags import TagsField from app.security import path_traversal_check from django.utils.translation import gettext_lazy as _ - +from webodm import settings def flatten_files(request_files): # MultiValueDict in, flat array of files out @@ -420,18 +422,52 @@ def post(self, request, project_pk=None): if import_url and len(files) > 0: raise exceptions.ValidationError(detail=_("Cannot create task, either specify a URL or upload 1 file.")) + chunk_index = request.data.get('dzchunkindex') + uuid = request.data.get('dzuuid') + total_chunk_count = request.data.get('dztotalchunkcount', None) + + # Chunked upload? + tmp_upload_file = None + if len(files) > 0 and chunk_index is not None and uuid is not None and total_chunk_count is not None: + byte_offset = request.data.get('dzchunkbyteoffset', 0) + + try: + chunk_index = int(chunk_index) + byte_offset = int(byte_offset) + total_chunk_count = int(total_chunk_count) + except ValueError: + raise exceptions.ValidationError(detail="chunkIndex is not an int") + uuid = re.sub('[^0-9a-zA-Z-]+', "", uuid) + + tmp_upload_file = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{uuid}.upload") + if os.path.isfile(tmp_upload_file) and chunk_index == 0: + os.unlink(tmp_upload_file) + + with open(tmp_upload_file, 'ab') as fd: + fd.seek(byte_offset) + if isinstance(files[0], InMemoryUploadedFile): + for chunk in files[0].chunks(): + fd.write(chunk) + else: + with open(files[0].temporary_file_path(), 'rb') as file: + fd.write(file.read()) + + if chunk_index + 1 < total_chunk_count: + return Response({'uploaded': True}, status=status.HTTP_200_OK) + + # Ready to import with transaction.atomic(): task = models.Task.objects.create(project=project, - auto_processing_node=False, - name=task_name, - import_url=import_url if import_url else "file://all.zip", - status=status_codes.RUNNING, - pending_action=pending_actions.IMPORT) + auto_processing_node=False, + name=task_name, + import_url=import_url if import_url else "file://all.zip", + status=status_codes.RUNNING, + pending_action=pending_actions.IMPORT) task.create_task_directories() + destination_file = task.assets_path("all.zip") - if len(files) > 0: - destination_file = task.assets_path("all.zip") - + # Non-chunked file import + if tmp_upload_file is None and len(files) > 0: with open(destination_file, 'wb+') as fd: if isinstance(files[0], InMemoryUploadedFile): for chunk in files[0].chunks(): @@ -439,6 +475,9 @@ def post(self, request, project_pk=None): else: with open(files[0].temporary_file_path(), 'rb') as file: copyfileobj(file, fd) + elif tmp_upload_file is not None: + # Move + shutil.move(tmp_upload_file, destination_file) worker_tasks.process_task.delay(task.id) diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx index 72dc4ec61..b3ed11dcb 100644 --- a/app/static/app/js/components/ImportTaskPanel.jsx +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -53,7 +53,8 @@ class ImportTaskPanel extends React.Component { clickable: this.uploadButton, chunkSize: 2147483647, timeout: 2147483647, - + chunking: true, + chunkSize: 16000000, // 16MB headers: { [csrf.header]: csrf.token } @@ -69,6 +70,7 @@ class ImportTaskPanel extends React.Component { this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0}); }) .on("uploadprogress", (file, progress, bytesSent) => { + if (progress == 100) return; // Workaround for chunked upload progress bar jumping around this.setState({ progress, totalBytes: file.size, From 9d336a5c6193829ba4360b249c24fa2d459ae93e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 15:45:57 -0400 Subject: [PATCH 56/95] Add unit test --- app/tests/test_api_task_import.py | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/app/tests/test_api_task_import.py b/app/tests/test_api_task_import.py index fb557501b..16b362e6b 100644 --- a/app/tests/test_api_task_import.py +++ b/app/tests/test_api_task_import.py @@ -166,3 +166,54 @@ def test_task(self): self.assertEqual(corrupted_task.status, status_codes.FAILED) self.assertTrue("Invalid" in corrupted_task.last_error) + # Test chunked upload import + assets_file = open(assets_path, 'rb') + assets_size = os.path.getsize(assets_path) + chunk_1_size = assets_size // 2 + chunk_1_path = os.path.join(os.path.dirname(assets_path), "1.zip") + chunk_2_path = os.path.join(os.path.dirname(assets_path), "2.zip") + with open(chunk_1_path, 'wb') as f: + assets_file.seek(0) + f.write(assets_file.read(chunk_1_size)) + with open(chunk_2_path, 'wb') as f: + f.write(assets_file.read()) + + chunk_1 = open(chunk_1_path, 'rb') + chunk_2 = open(chunk_2_path, 'rb') + assets_file.close() + + res = client.post("/api/projects/{}/tasks/import".format(project.id), { + 'file': [chunk_1], + 'dzuuid': 'abc-test', + 'dzchunkindex': 0, + 'dztotalchunkcount': 2, + 'dzchunkbyteoffset': 0 + }, format="multipart") + self.assertEqual(res.status_code, status.HTTP_200_OK) + self.assertTrue(res.data['uploaded']) + chunk_1.close() + + res = client.post("/api/projects/{}/tasks/import".format(project.id), { + 'file': [chunk_2], + 'dzuuid': 'abc-test', + 'dzchunkindex': 1, + 'dztotalchunkcount': 2, + 'dzchunkbyteoffset': chunk_1_size + }, format="multipart") + self.assertEqual(res.status_code, status.HTTP_201_CREATED) + chunk_2.close() + + file_import_task = Task.objects.get(id=res.data['id']) + # Wait for completion + c = 0 + while c < 10: + worker.tasks.process_pending_tasks() + file_import_task.refresh_from_db() + if file_import_task.status == status_codes.COMPLETED: + break + c += 1 + time.sleep(1) + + self.assertEqual(file_import_task.import_url, "file://all.zip") + self.assertEqual(file_import_task.images_count, 1) + From a364de217645f5660b6901717ca407e8720a90b5 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 15:54:02 -0400 Subject: [PATCH 57/95] Better validation error msg --- app/api/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index 26f612c77..b80bb90a7 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -436,7 +436,7 @@ def post(self, request, project_pk=None): byte_offset = int(byte_offset) total_chunk_count = int(total_chunk_count) except ValueError: - raise exceptions.ValidationError(detail="chunkIndex is not an int") + raise exceptions.ValidationError(detail="some parameters are not integers") uuid = re.sub('[^0-9a-zA-Z-]+', "", uuid) tmp_upload_file = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{uuid}.upload") From 9cf533f87c99dba69dcb364584b258c3bdad4149 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 15:54:19 -0400 Subject: [PATCH 58/95] capitalize --- app/api/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index b80bb90a7..9cb56c4b4 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -436,7 +436,7 @@ def post(self, request, project_pk=None): byte_offset = int(byte_offset) total_chunk_count = int(total_chunk_count) except ValueError: - raise exceptions.ValidationError(detail="some parameters are not integers") + raise exceptions.ValidationError(detail="Some parameters are not integers") uuid = re.sub('[^0-9a-zA-Z-]+', "", uuid) tmp_upload_file = os.path.join(settings.FILE_UPLOAD_TEMP_DIR, f"{uuid}.upload") From cdeae25426b8d7f26d98712ec02625225d723eea Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 16:05:08 -0400 Subject: [PATCH 59/95] Build webpack with production --- webpack.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webpack.config.js b/webpack.config.js index 2f0b53633..4d752a17d 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -5,7 +5,7 @@ let ExtractTextPlugin = require('extract-text-webpack-plugin'); let LiveReloadPlugin = require('webpack-livereload-plugin'); module.exports = { - mode: 'development', + mode: 'production', context: __dirname, entry: { From 9ece192f28483738b3d3be5f55a3392926fed8af Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 16:50:28 -0400 Subject: [PATCH 60/95] Conditional webpack mode --- webpack.config.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/webpack.config.js b/webpack.config.js index 4d752a17d..b2be6e149 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -4,10 +4,13 @@ let BundleTracker = require('webpack-bundle-tracker'); let ExtractTextPlugin = require('extract-text-webpack-plugin'); let LiveReloadPlugin = require('webpack-livereload-plugin'); +const mode = process.argv.indexOf("production") !== -1 ? "production" : "development"; +console.log(`Webpack mode: ${mode}`); + module.exports = { - mode: 'production', + mode, context: __dirname, - + entry: { main: ['./app/static/app/js/main.jsx'], Console: ['./app/static/app/js/Console.jsx'], From ef4db8f4910b18fee910f18595bec2cecf259156 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 17:20:14 -0400 Subject: [PATCH 61/95] Disable rasterio warnings --- app/api/tiler.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/api/tiler.py b/app/api/tiler.py index d283acae7..91a496e14 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -3,6 +3,7 @@ from rasterio.enums import ColorInterp from rasterio.crs import CRS from rasterio.features import bounds as featureBounds +from rasterio.errors import NotGeoreferencedWarning import urllib import os from .common import get_asset_download_filename @@ -16,7 +17,7 @@ from rio_tiler.profiles import img_profiles from rio_tiler.colormap import cmap as colormap, apply_cmap from rio_tiler.io import COGReader -from rio_tiler.errors import InvalidColorMapName +from rio_tiler.errors import InvalidColorMapName, AlphaBandWarning import numpy as np from .custom_colormaps_helper import custom_colormaps from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS @@ -28,7 +29,13 @@ from rest_framework.response import Response from worker.tasks import export_raster, export_pointcloud from django.utils.translation import gettext as _ +import warnings +# Disable: NotGeoreferencedWarning: Dataset has no geotransform, gcps, or rpcs. The identity matrix be returned. +warnings.filterwarnings("ignore", category=NotGeoreferencedWarning) + +# Disable: Alpha band was removed from the output data array +warnings.filterwarnings("ignore", category=AlphaBandWarning) for custom_colormap in custom_colormaps: colormap = colormap.register(custom_colormap) From 92b98389ada38ed65f23a6a1c352f9d221e833e3 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Sep 2023 17:38:03 -0400 Subject: [PATCH 62/95] Faster map initialization --- app/static/app/js/components/Map.jsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index 810b66661..b93ff5611 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -482,7 +482,11 @@ _('Example:'), }); new AddOverlayCtrl().addTo(this.map); - this.map.fitWorld(); + this.map.fitBounds([ + [13.772919746115805, + 45.664640939831735], + [13.772825784981254, + 45.664591558975154]]); this.map.attributionControl.setPrefix(""); this.setState({showLoading: true}); From 0093ca71cddff6becd084f6fddd7ad82a7bf7c38 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 22 Sep 2023 15:34:27 -0400 Subject: [PATCH 63/95] Fix encoding on Windows --- app/classes/console.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/classes/console.py b/app/classes/console.py index 96e55ed0a..8de9296ec 100644 --- a/app/classes/console.py +++ b/app/classes/console.py @@ -16,7 +16,7 @@ def __str__(self): return "" try: - with open(self.file, 'r') as f: + with open(self.file, 'r', encoding="utf-8") as f: return f.read() except IOError: logger.warn("Cannot read console file: %s" % self.file) @@ -36,7 +36,7 @@ def append(self, text): if not os.path.isdir(self.base_dir): os.makedirs(self.base_dir, exist_ok=True) - with open(self.file, "a") as f: + with open(self.file, "a", encoding="utf-8") as f: f.write(text) except IOError: logger.warn("Cannot append to console file: %s" % self.file) @@ -47,7 +47,7 @@ def reset(self, text = ""): if not os.path.isdir(self.base_dir): os.makedirs(self.base_dir, exist_ok=True) - with open(self.file, "w") as f: + with open(self.file, "w", encoding="utf-8") as f: f.write(text) except IOError: logger.warn("Cannot reset console file: %s" % self.file) From 1b327fb56edfcfb1d2d0de21722ff4047a1ccb4b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:04:04 -0400 Subject: [PATCH 64/95] GDAL based contours --- coreplugins/contours/api.py | 84 ++++++++++++++++++++++++++++--------- worker/tasks.py | 2 - 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/coreplugins/contours/api.py b/coreplugins/contours/api.py index 7d07b14e5..d6c88eaed 100644 --- a/coreplugins/contours/api.py +++ b/coreplugins/contours/api.py @@ -3,10 +3,68 @@ from rest_framework import status from rest_framework.response import Response from app.plugins.views import TaskView, CheckTask, GetTaskResult -from worker.tasks import execute_grass_script -from app.plugins.grass_engine import grass, GrassEngineException, cleanup_grass_context +from app.plugins.worker import run_function_async from django.utils.translation import gettext_lazy as _ +class ContoursException(Exception): + pass + +def calc_contours(dem, epsg, interval, output_format, simplify): + import os + import subprocess + import tempfile + import shutil + from webodm import settings + + ext = "" + if output_format == "GeoJSON": + ext = "json" + elif output_format == "GPKG": + ext = "gpkg" + elif output_format == "DXF": + ext = "dxf" + elif output_format == "ESRI Shapefile": + ext = "shp" + MIN_CONTOUR_LENGTH = 10 + + tmpdir = os.path.join(settings.MEDIA_TMP, os.path.basename(tempfile.mkdtemp('_contours', dir=settings.MEDIA_TMP))) + gdal_contour_bin = shutil.which("gdal_contour") + ogr2ogr_bin = shutil.which("ogr2ogr") + + if gdal_contour_bin is None: + return {'error': 'Cannot find gdal_contour'} + if ogr2ogr_bin is None: + return {'error': 'Cannot find ogr2ogr'} + + contours_file = f"contours.gpkg" + p = subprocess.Popen([gdal_contour_bin, "-q", "-a", "level", "-3d", "-f", "GPKG", "-i", str(interval), dem, contours_file], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + out = out.decode('utf-8').strip() + err = err.decode('utf-8').strip() + success = p.returncode == 0 + + if not success: + return {'error', f'Error calling gdal_contour: {str(err)}'} + + outfile = os.path.join(tmpdir, f"output.{ext}") + p = subprocess.Popen([ogr2ogr_bin, outfile, contours_file, "-simplify", str(simplify), "-f", output_format, "-t_srs", f"EPSG:{epsg}", + "-dialect", "sqlite", "-sql", f"SELECT * FROM contour WHERE ST_Length(GEOM) >= {MIN_CONTOUR_LENGTH}"], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + out = out.decode('utf-8').strip() + err = err.decode('utf-8').strip() + success = p.returncode == 0 + + if not success: + return {'error', f'Error calling ogr2ogr: {str(err)}'} + + if not os.path.isfile(outfile): + return {'error': f'Cannot find output file: {outfile}'} + + return {'file': outfile} + + class TaskContoursGenerate(TaskView): def post(self, request, pk=None): task = self.get_and_check_task(request, pk) @@ -23,36 +81,24 @@ def post(self, request, pk=None): elif layer == 'DTM': dem = os.path.abspath(task.get_asset_download_path("dtm.tif")) else: - raise GrassEngineException('{} is not a valid layer.'.format(layer)) + raise ContoursException('{} is not a valid layer.'.format(layer)) - context = grass.create_context({'auto_cleanup' : False}) epsg = int(request.data.get('epsg', '3857')) interval = float(request.data.get('interval', 1)) format = request.data.get('format', 'GPKG') supported_formats = ['GPKG', 'ESRI Shapefile', 'DXF', 'GeoJSON'] if not format in supported_formats: - raise GrassEngineException("Invalid format {} (must be one of: {})".format(format, ",".join(supported_formats))) + raise ContoursException("Invalid format {} (must be one of: {})".format(format, ",".join(supported_formats))) simplify = float(request.data.get('simplify', 0.01)) - context.add_param('dem_file', dem) - context.add_param('interval', interval) - context.add_param('format', format) - context.add_param('simplify', simplify) - context.add_param('epsg', epsg) - context.set_location(dem) - - celery_task_id = execute_grass_script.delay(os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "calc_contours.py" - ), context.serialize(), 'file').task_id - + celery_task_id = run_function_async(calc_contours, dem, epsg, interval, format, simplify).task_id return Response({'celery_task_id': celery_task_id}, status=status.HTTP_200_OK) - except GrassEngineException as e: + except ContoursException as e: return Response({'error': str(e)}, status=status.HTTP_200_OK) class TaskContoursCheck(CheckTask): def on_error(self, result): - cleanup_grass_context(result['context']) + pass def error_check(self, result): contours_file = result.get('file') diff --git a/worker/tasks.py b/worker/tasks.py index 12ff4f1e6..392582b79 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -175,7 +175,6 @@ def execute_grass_script(script, serialized_context = {}, out_key='output'): logger.error(str(e)) return {'error': str(e), 'context': ctx.serialize()} - @app.task(bind=True) def export_raster(self, input, **opts): try: @@ -238,4 +237,3 @@ def check_quotas(): break else: p.clear_quota_deadline() - From c8c0f518052e587c3d7303a7d6c250161271b418 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:08:11 -0400 Subject: [PATCH 65/95] Assign layer name --- coreplugins/contours/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreplugins/contours/api.py b/coreplugins/contours/api.py index d6c88eaed..2c4a0d1e5 100644 --- a/coreplugins/contours/api.py +++ b/coreplugins/contours/api.py @@ -48,7 +48,7 @@ def calc_contours(dem, epsg, interval, output_format, simplify): return {'error', f'Error calling gdal_contour: {str(err)}'} outfile = os.path.join(tmpdir, f"output.{ext}") - p = subprocess.Popen([ogr2ogr_bin, outfile, contours_file, "-simplify", str(simplify), "-f", output_format, "-t_srs", f"EPSG:{epsg}", + p = subprocess.Popen([ogr2ogr_bin, outfile, contours_file, "-simplify", str(simplify), "-f", output_format, "-t_srs", f"EPSG:{epsg}", "-nln", "contours", "-dialect", "sqlite", "-sql", f"SELECT * FROM contour WHERE ST_Length(GEOM) >= {MIN_CONTOUR_LENGTH}"], cwd=tmpdir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() From 0e7d9ee6f2d38c9471e8ad03de03c9b8d492bbac Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:20:27 -0400 Subject: [PATCH 66/95] Shapefile support --- coreplugins/contours/api.py | 12 ++++ coreplugins/contours/calc_contours.py | 93 --------------------------- 2 files changed, 12 insertions(+), 93 deletions(-) delete mode 100755 coreplugins/contours/calc_contours.py diff --git a/coreplugins/contours/api.py b/coreplugins/contours/api.py index 2c4a0d1e5..603bc2175 100644 --- a/coreplugins/contours/api.py +++ b/coreplugins/contours/api.py @@ -14,6 +14,7 @@ def calc_contours(dem, epsg, interval, output_format, simplify): import subprocess import tempfile import shutil + import glob from webodm import settings ext = "" @@ -61,6 +62,17 @@ def calc_contours(dem, epsg, interval, output_format, simplify): if not os.path.isfile(outfile): return {'error': f'Cannot find output file: {outfile}'} + + if output_format == "ESRI Shapefile": + ext="zip" + shp_dir = os.path.join(tmpdir, "contours") + os.makedirs(shp_dir) + contour_files = glob.glob(os.path.join(tmpdir, "output.*")) + for cf in contour_files: + shutil.move(cf, shp_dir) + + shutil.make_archive(os.path.join(tmpdir, 'output'), 'zip', shp_dir) + outfile = os.path.join(tmpdir, f"output.{ext}") return {'file': outfile} diff --git a/coreplugins/contours/calc_contours.py b/coreplugins/contours/calc_contours.py deleted file mode 100755 index 987ab5bb5..000000000 --- a/coreplugins/contours/calc_contours.py +++ /dev/null @@ -1,93 +0,0 @@ -#%module -#% description: Calculate contours -#%end -#%option -#% key: dem_file -#% type: string -#% required: yes -#% multiple: no -#% description: GeoTIFF DEM containing the surface to calculate contours -#%end -#%option -#% key: interval -#% type: double -#% required: yes -#% multiple: no -#% description: Contours interval -#%end -#%option -#% key: format -#% type: string -#% required: yes -#% multiple: no -#% description: OGR output format -#%end -#%option -#% key: simplify -#% type: double -#% required: yes -#% multiple: no -#% description: OGR output format -#%end -#%option -#% key: epsg -#% type: string -#% required: yes -#% multiple: no -#% description: target EPSG code -#%end - -# output: If successful, prints the full path to the contours file. Otherwise it prints "error" - -import sys -import glob -import os -import shutil -from grass.pygrass.modules import Module -import grass.script as grass -import subprocess - -def main(): - ext = "" - if opts['format'] == "GeoJSON": - ext = "json" - elif opts['format'] == "GPKG": - ext = "gpkg" - elif opts['format'] == "DXF": - ext = "dxf" - elif opts['format'] == "ESRI Shapefile": - ext = "shp" - - MIN_CONTOUR_LENGTH = 5 - Module("r.external", input=opts['dem_file'], output="dem", overwrite=True) - Module("g.region", raster="dem") - Module("r.contour", input="dem", output="contours", step=opts["interval"], overwrite=True) - Module("v.generalize", input="contours", output="contours_smooth", method="douglas", threshold=opts["simplify"], overwrite=True) - Module("v.generalize", input="contours_smooth", output="contours_simplified", method="chaiken", threshold=1, overwrite=True) - Module("v.generalize", input="contours_simplified", output="contours_final", method="douglas", threshold=opts["simplify"], overwrite=True) - Module("v.edit", map="contours_final", tool="delete", threshold=[-1,0,-MIN_CONTOUR_LENGTH], query="length") - Module("v.out.ogr", input="contours_final", output="temp.gpkg", format="GPKG") - - subprocess.check_call(["ogr2ogr", "-t_srs", "EPSG:%s" % opts['epsg'], - '-overwrite', '-f', opts["format"], "output.%s" % ext, "temp.gpkg"], stdout=subprocess.DEVNULL) - - if os.path.isfile("output.%s" % ext): - if opts["format"] == "ESRI Shapefile": - ext="zip" - os.makedirs("contours") - contour_files = glob.glob("output.*") - for cf in contour_files: - shutil.move(cf, os.path.join("contours", os.path.basename(cf))) - - shutil.make_archive('output', 'zip', 'contours/') - - print(os.path.join(os.getcwd(), "output.%s" % ext)) - else: - print("error") - - return 0 - -if __name__ == "__main__": - opts, _ = grass.parser() - sys.exit(main()) - From a852dfb04e714e00918f3a32876e0ca11ffaab2c Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 26 Sep 2023 12:37:10 -0400 Subject: [PATCH 67/95] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 70411580f..8c812941d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.2", + "version": "2.1.3", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { From 0501938d6138f41d0c20d7aa0d617465e53dc248 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 27 Sep 2023 16:43:25 -0400 Subject: [PATCH 68/95] Disable logging --- nginx/nginx-ssl.conf.template | 4 ++-- nginx/nginx.conf.template | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nginx/nginx-ssl.conf.template b/nginx/nginx-ssl.conf.template index c630bf011..d27dbf3d9 100644 --- a/nginx/nginx-ssl.conf.template +++ b/nginx/nginx-ssl.conf.template @@ -3,7 +3,7 @@ worker_processes 1; # Change this if running outside docker! user root root; pid /tmp/nginx.pid; -error_log /tmp/nginx.error.log; +error_log /dev/null; events { worker_connections 1024; # increase if you have lots of clients @@ -16,7 +16,7 @@ http { # fallback in case we can't determine a type default_type application/octet-stream; - access_log /tmp/nginx.access.log combined; + access_log off; sendfile on; upstream app_server { diff --git a/nginx/nginx.conf.template b/nginx/nginx.conf.template index a2b2699a6..9495d096a 100644 --- a/nginx/nginx.conf.template +++ b/nginx/nginx.conf.template @@ -3,7 +3,7 @@ worker_processes 1; # Change this if running outside docker! user root root; pid /tmp/nginx.pid; -error_log /tmp/nginx.error.log; +error_log /dev/null; events { worker_connections 1024; # increase if you have lots of clients @@ -16,7 +16,7 @@ http { # fallback in case we can't determine a type default_type application/octet-stream; - access_log /tmp/nginx.access.log combined; + access_log off; sendfile on; upstream app_server { From d9736cf11fc1833a47ab47bab475e7ded42fa8a2 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 28 Sep 2023 16:48:47 -0400 Subject: [PATCH 69/95] Add borg backup media pattern generator --- app/management/commands/borg.py | 63 +++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 app/management/commands/borg.py diff --git a/app/management/commands/borg.py b/app/management/commands/borg.py new file mode 100644 index 000000000..60b0d1a11 --- /dev/null +++ b/app/management/commands/borg.py @@ -0,0 +1,63 @@ +import os +from django.core.management.base import BaseCommand +from django.core.management import call_command +from app.models import Project +from webodm import settings + +class Command(BaseCommand): + requires_system_checks = [] + + def add_arguments(self, parser): + parser.add_argument("action", type=str, choices=['mediapattern']) + parser.add_argument("--skip-images", action='store_true', required=False, help="Skip images") + parser.add_argument("--skip-no-quotas", action='store_true', required=False, help="Skip directories owned by users with no quota (0)") + parser.add_argument("--skip-tiles", action='store_true', required=False, help="Skip tiled assets which can be regenerated from other data") + parser.add_argument("--skip-legacy-textured-models", action='store_true', required=False, help="Skip textured models in OBJ format") + + super(Command, self).add_arguments(parser) + + def handle(self, **options): + if options.get('action') == 'mediapattern': + print("# BorgBackup pattern file for media directory") + print("# Generated with WebODM") + print("") + + print("# Skip anything but project folder") + for d in os.listdir(settings.MEDIA_ROOT): + if d != "project": + print(f"! {d}") + + if options.get('skip_no_quotas'): + skip_projects = Project.objects.filter(owner__profile__quota=0).order_by('id') + else: + skip_projects = [] + + print("") + print("# Skip projects") + for sp in skip_projects: + print("- " + os.path.join("project", str(sp.id))) + + if options.get('skip_images'): + print("") + print("# Skip images/other files") + print("- project/*/task/*/*.*") + + if options.get('skip_tiles'): + print("") + print("# Skip entwine/potree folders") + print("! project/*/task/*/assets/entwine_pointcloud") + print("! project/*/task/*/assets/potree_pointcloud") + print("") + print("# Skip tiles folders") + print("! project/*/task/*/assets/*_tiles") + + print("# Skip data") + print("! project/*/task/*/data") + + if options.get('skip_legacy_textured_models'): + print("") + print("# Skip OBJ texture model files") + print("+ project/*/task/*/assets/odm_texturing/*.glb") + print("- project/*/task/*/assets/odm_texturing") + + From d5e597fee86a91c761ae4408a1d4d639b9600894 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 2 Oct 2023 10:10:48 -0400 Subject: [PATCH 70/95] Update formulas --- app/api/formulas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/api/formulas.py b/app/api/formulas.py index cdfe95f56..f6855cdf0 100644 --- a/app/api/formulas.py +++ b/app/api/formulas.py @@ -146,11 +146,12 @@ 'RGBN', 'GRReN', + 'RGBNRe', 'BGRNRe', 'BGRReN', - 'RGBNRe', 'RGBReN', + 'RGBNReL', 'BGRNReL', 'BGRReNL', From 474e2d844b8e8fc60de4d455e2357ac170517fea Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 2 Oct 2023 10:19:16 -0400 Subject: [PATCH 71/95] Add RGNRe --- app/api/formulas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/formulas.py b/app/api/formulas.py index f6855cdf0..457303200 100644 --- a/app/api/formulas.py +++ b/app/api/formulas.py @@ -144,6 +144,7 @@ 'NRB', 'RGBN', + 'RGNRe', 'GRReN', 'RGBNRe', From 530720b6994a1f8132d5812af5217291f4ed33b1 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:13:56 -0400 Subject: [PATCH 72/95] Adds support for automatically selecting the proper band filter --- app/api/formulas.py | 52 +++++++++++++++++-- app/api/tiler.py | 20 ++++++- app/migrations/0039_task_orthophoto_bands.py | 43 +++++++++++++++ app/models/task.py | 7 ++- .../app/js/components/LayersControlLayer.jsx | 14 +++-- app/static/app/js/components/Map.jsx | 14 +++++ app/tests/test_api_task.py | 2 +- app/tests/test_formulas.py | 2 +- locale | 2 +- 9 files changed, 141 insertions(+), 15 deletions(-) create mode 100644 app/migrations/0039_task_orthophoto_bands.py diff --git a/app/api/formulas.py b/app/api/formulas.py index 457303200..21d4e2971 100644 --- a/app/api/formulas.py +++ b/app/api/formulas.py @@ -156,6 +156,8 @@ 'BGRNReL', 'BGRReNL', + 'RGBNRePL', + 'L', # FLIR camera has a single LWIR band # more? @@ -171,7 +173,7 @@ def lookup_formula(algo, band_order = 'RGB'): if algo not in algos: raise ValueError("Cannot find algorithm " + algo) - + input_bands = tuple(b for b in re.split(r"([A-Z][a-z]*)", band_order) if b != "") def repl(matches): @@ -193,7 +195,7 @@ def get_algorithm_list(max_bands=3): if k.startswith("_"): continue - cam_filters = get_camera_filters_for(algos[k], max_bands) + cam_filters = get_camera_filters_for(algos[k]['expr'], max_bands) if len(cam_filters) == 0: continue @@ -206,9 +208,9 @@ def get_algorithm_list(max_bands=3): return res -def get_camera_filters_for(algo, max_bands=3): +@lru_cache(maxsize=100) +def get_camera_filters_for(expr, max_bands=3): result = [] - expr = algo['expr'] pattern = re.compile("([A-Z]+?[a-z]*)") bands = list(set(re.findall(pattern, expr))) for f in camera_filters: @@ -226,3 +228,45 @@ def get_camera_filters_for(algo, max_bands=3): return result +@lru_cache(maxsize=1) +def get_bands_lookup(): + bands_aliases = { + 'R': ['red', 'r'], + 'G': ['green', 'g'], + 'B': ['blue', 'b'], + 'N': ['nir', 'n'], + 'Re': ['rededge', 're'], + 'P': ['panchro', 'p'], + 'L': ['lwir', 'l'] + } + bands_lookup = {} + for band in bands_aliases: + for a in bands_aliases[band]: + bands_lookup[a] = band + return bands_lookup + +def get_auto_bands(orthophoto_bands, formula): + algo = algos.get(formula) + if not algo: + raise ValueError("Cannot find formula: " + formula) + + max_bands = len(orthophoto_bands) - 1 # minus alpha + filters = get_camera_filters_for(algo['expr'], max_bands) + if not filters: + raise valueError(f"Cannot find filters for {algo} with max bands {max_bands}") + + bands_lookup = get_bands_lookup() + band_order = "" + + for band in orthophoto_bands: + if band['name'] == 'alpha' or (not band['description']): + continue + f_band = bands_lookup.get(band['description'].lower()) + + if f_band is not None: + band_order += f_band + + if band_order in filters: + return band_order, True + else: + return filters[0], False # Fallback diff --git a/app/api/tiler.py b/app/api/tiler.py index 91a496e14..855eef569 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -23,7 +23,7 @@ from app.raster_utils import extension_for_export_format, ZOOM_EXTRA_LEVELS from .hsvblend import hsv_blend from .hillshade import LightSource -from .formulas import lookup_formula, get_algorithm_list +from .formulas import lookup_formula, get_algorithm_list, get_auto_bands from .tasks import TaskNestedView from rest_framework import exceptions from rest_framework.response import Response @@ -141,6 +141,12 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): if boundaries_feature == '': boundaries_feature = None if boundaries_feature is not None: boundaries_feature = json.loads(boundaries_feature) + + is_auto_bands_match = False + is_auto_bands = False + if bands == 'auto' and formula: + is_auto_bands = True + bands, is_auto_bands_match = get_auto_bands(task.orthophoto_bands, formula) try: expr, hrange = lookup_formula(formula, bands) if defined_range is not None: @@ -224,6 +230,8 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): colormaps = [] algorithms = [] + auto_bands = {'filter': '', 'match': None} + if tile_type in ['dsm', 'dtm']: colormaps = ['viridis', 'jet', 'terrain', 'gist_earth', 'pastel1'] elif formula and bands: @@ -231,9 +239,14 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): 'better_discrete_ndvi', 'viridis', 'plasma', 'inferno', 'magma', 'cividis', 'jet', 'jet_r'] algorithms = *get_algorithm_list(band_count), + if is_auto_bands: + auto_bands['filter'] = bands + auto_bands['match'] = is_auto_bands_match info['color_maps'] = [] info['algorithms'] = algorithms + info['auto_bands'] = auto_bands + if colormaps: for cmap in colormaps: try: @@ -254,6 +267,7 @@ def get(self, request, pk=None, project_pk=None, tile_type=""): info['maxzoom'] += ZOOM_EXTRA_LEVELS info['minzoom'] -= ZOOM_EXTRA_LEVELS info['bounds'] = {'value': src.bounds, 'crs': src.dataset.crs} + return Response(info) @@ -296,6 +310,8 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", if color_map == '': color_map = None if hillshade == '' or hillshade == '0': hillshade = None if tilesize == '' or tilesize is None: tilesize = 256 + if bands == 'auto' and formula: + bands, _ = get_auto_bands(task.orthophoto_bands, formula) try: tilesize = int(tilesize) @@ -611,4 +627,4 @@ def post(self, request, pk=None, project_pk=None, asset_type=None): else: celery_task_id = export_pointcloud.delay(url, epsg=epsg, format=export_format).task_id - return Response({'celery_task_id': celery_task_id, 'filename': filename}) \ No newline at end of file + return Response({'celery_task_id': celery_task_id, 'filename': filename}) diff --git a/app/migrations/0039_task_orthophoto_bands.py b/app/migrations/0039_task_orthophoto_bands.py new file mode 100644 index 000000000..c801ab853 --- /dev/null +++ b/app/migrations/0039_task_orthophoto_bands.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2.27 on 2023-10-02 10:21 + +import rasterio +import os +import django.contrib.postgres.fields.jsonb +from django.db import migrations +from webodm import settings + +def update_orthophoto_bands_fields(apps, schema_editor): + Task = apps.get_model('app', 'Task') + + for t in Task.objects.all(): + + bands = [] + orthophoto_path = os.path.join(settings.MEDIA_ROOT, "project", str(t.project.id), "task", str(t.id), "assets", "odm_orthophoto", "odm_orthophoto.tif") + + if os.path.isfile(orthophoto_path): + try: + with rasterio.open(orthophoto_path) as f: + names = [c.name for c in f.colorinterp] + for i, n in enumerate(names): + bands.append({ + 'name': n, + 'description': f.descriptions[i] + }) + except Exception as e: + print(e) + + print("Updating {} (with orthophoto bands: {})".format(t, str(bands))) + + t.orthophoto_bands = bands + t.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0038_remove_task_console_output'), + ] + + operations = [ + migrations.RunPython(update_orthophoto_bands_fields), + ] diff --git a/app/models/task.py b/app/models/task.py index 1f0c4c0c6..89bcb85c0 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -1024,7 +1024,12 @@ def update_orthophoto_bands_field(self, commit=False): if os.path.isfile(orthophoto_path): with rasterio.open(orthophoto_path) as f: - bands = [c.name for c in f.colorinterp] + names = [c.name for c in f.colorinterp] + for i, n in enumerate(names): + bands.append({ + 'name': n, + 'description': f.descriptions[i] + }) self.orthophoto_bands = bands if commit: self.save() diff --git a/app/static/app/js/components/LayersControlLayer.jsx b/app/static/app/js/components/LayersControlLayer.jsx index 36509ad53..2aaaf531a 100644 --- a/app/static/app/js/components/LayersControlLayer.jsx +++ b/app/static/app/js/components/LayersControlLayer.jsx @@ -134,7 +134,7 @@ export default class LayersControlLayer extends React.Component { // Check if bands need to be switched const algo = this.getAlgorithm(e.target.value); - if (algo && algo['filters'].indexOf(bands) === -1) bands = algo['filters'][0]; // Pick first + if (algo && algo['filters'].indexOf(bands) === -1 && bands !== "auto") bands = algo['filters'][0]; // Pick first this.setState({formula: e.target.value, bands}); } @@ -262,7 +262,7 @@ export default class LayersControlLayer extends React.Component { render(){ const { colorMap, bands, hillshade, formula, histogramLoading, exportLoading } = this.state; const { meta, tmeta } = this; - const { color_maps, algorithms } = tmeta; + const { color_maps, algorithms, auto_bands } = tmeta; const algo = this.getAlgorithm(formula); let cmapValues = null; @@ -298,13 +298,17 @@ export default class LayersControlLayer extends React.Component { {bands !== "" && algo ?
    - +
    {histogramLoading ? : - + {algo.filters.map(f => )} - } + , + bands == "auto" && !auto_bands.match ? + + : ""]}
    : ""} diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index b93ff5611..1e78675c4 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -94,6 +94,16 @@ class Map extends React.Component { return ""; } + hasBands = (bands, orthophoto_bands) => { + if (!orthophoto_bands) return false; + console.log(orthophoto_bands) + for (let i = 0; i < bands.length; i++){ + if (orthophoto_bands.find(b => b.description !== null && b.description.toLowerCase() === bands[i].toLowerCase()) === undefined) return false; + } + + return true; + } + loadImageryLayers(forceAddLayers = false){ // Cancel previous requests if (this.tileJsonRequests) { @@ -131,7 +141,11 @@ class Map extends React.Component { // Single band, probably thermal dataset, in any case we can't render NDVI // because it requires 3 bands metaUrl += "?formula=Celsius&bands=L&color_map=magma"; + }else if (meta.task && meta.task.orthophoto_bands){ + let formula = this.hasBands(["red", "green", "nir"], meta.task.orthophoto_bands) ? "NDVI" : "VARI"; + metaUrl += `?formula=${formula}&bands=auto&color_map=rdylgn`; }else{ + // This should never happen? metaUrl += "?formula=NDVI&bands=RGN&color_map=rdylgn"; } }else if (type == "dsm" || type == "dtm"){ diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index 1abbff4ec..bf3af575a 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -679,7 +679,7 @@ def test_task(self): for k in algos: a = algos[k] - filters = get_camera_filters_for(a) + filters = get_camera_filters_for(a['expr']) for f in filters: params.append(("orthophoto", "formula={}&bands={}&color_map=rdylgn".format(k, f), status.HTTP_200_OK)) diff --git a/app/tests/test_formulas.py b/app/tests/test_formulas.py index 76077d47a..ab9117da0 100644 --- a/app/tests/test_formulas.py +++ b/app/tests/test_formulas.py @@ -38,7 +38,7 @@ def test_algo_list(self): bands = list(set(re.findall(pattern, f))) self.assertTrue(len(bands) <= 3) - self.assertTrue(get_camera_filters_for(algos['VARI']) == ['RGB']) + self.assertTrue(get_camera_filters_for(algos['VARI']['expr']) == ['RGB']) # Request algorithms with more band filters al = get_algorithm_list(max_bands=5) diff --git a/locale b/locale index d253dd577..0a68f6ed5 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit d253dd5770c42a0705d0f861db3314b08f230a68 +Subproject commit 0a68f6ed5172a8838571a9872bcc3cb4f310794c From 43b24eb8b61e41b72180fe2eb51568fc4a873151 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:19:28 -0400 Subject: [PATCH 73/95] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8c812941d..1018c1b4b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "2.1.3", + "version": "2.2.0", "description": "User-friendly, extendable application and API for processing aerial imagery.", "main": "index.js", "scripts": { From b68e622234604e9ffda361672013ff450f78273b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:31:06 -0400 Subject: [PATCH 74/95] Update locales --- .../app/js/components/LayersControlLayer.jsx | 4 +- app/static/app/js/components/Map.jsx | 2 +- .../app/js/components/ProjectListItem.jsx | 2 + .../app/js/translations/odm_autogenerated.js | 152 +++++++++--------- locale | 2 +- 5 files changed, 82 insertions(+), 80 deletions(-) diff --git a/app/static/app/js/components/LayersControlLayer.jsx b/app/static/app/js/components/LayersControlLayer.jsx index 2aaaf531a..4996b01fd 100644 --- a/app/static/app/js/components/LayersControlLayer.jsx +++ b/app/static/app/js/components/LayersControlLayer.jsx @@ -302,12 +302,12 @@ export default class LayersControlLayer extends React.Component {
    {histogramLoading ? : - [ {algo.filters.map(f => )} , bands == "auto" && !auto_bands.match ? - + : ""]}
    : ""} diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index 1e78675c4..2d6caacc4 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -96,7 +96,7 @@ class Map extends React.Component { hasBands = (bands, orthophoto_bands) => { if (!orthophoto_bands) return false; - console.log(orthophoto_bands) + for (let i = 0; i < bands.length; i++){ if (orthophoto_bands.find(b => b.description !== null && b.description.toLowerCase() === bands[i].toLowerCase()) === undefined) return false; } diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 31d6ec429..51023e933 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -573,6 +573,8 @@ class ProjectListItem extends React.Component { render() { const { refreshing, data, filterTags } = this.state; const numTasks = data.tasks.length; + + // const numValidTasks = data.tasks.filter(t => ) const canEdit = this.hasPermission("change"); const userTags = Tags.userTags(data.tags); let deleteWarning = _("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?"); diff --git a/app/static/app/js/translations/odm_autogenerated.js b/app/static/app/js/translations/odm_autogenerated.js index bb87910bb..cd078b718 100644 --- a/app/static/app/js/translations/odm_autogenerated.js +++ b/app/static/app/js/translations/odm_autogenerated.js @@ -1,94 +1,94 @@ // Auto-generated with extract_odm_strings.py, do not edit! -_("show this help message and exit"); -_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); -_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); -_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); -_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); -_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); -_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); -_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); -_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); -_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); -_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); -_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); -_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); +_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); +_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); +_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); _("Skip generation of the orthophoto. This can save time if you only need 3D results or DEMs. Default: %(default)s"); +_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); +_("Perform image matching with the nearest images based on GPS exif data. Set to 0 to match by triangulation. Default: %(default)s"); +_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); _("Use this tag to build a DTM (Digital Terrain Model, ground only) using a simple morphological filter. Check the --dem* and --smrf* parameters for finer tuning. Default: %(default)s"); -_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); -_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); -_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); -_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); -_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); +_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); _("Set the radiometric calibration to perform on images. When processing multispectral and thermal images you should set this option to obtain reflectance/temperature values (otherwise you will get digital number values). [camera] applies black level, vignetting, row gradient gain/exposure compensation (if appropriate EXIF tags are found) and computes absolute temperature values. [camera+sun] is experimental, applies all the corrections of [camera], plus compensates for spectral radiance registered via a downwelling light sensor (DLS) taking in consideration the angle of the sun. Can be one of: %(choices)s. Default: %(default)s"); -_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); -_("Simple Morphological Filter slope parameter (rise over run). Default: %(default)s"); -_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); -_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); -_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); -_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); -_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); +_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); +_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); _("Filters the point cloud by keeping only a single point around a radius N (in meters). This can be useful to limit the output resolution of the point cloud and remove duplicate points. Set to 0 to disable sampling. Default: %(default)s"); -_("Copy output results to this folder after processing."); -_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); -_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); -_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); +_("End processing at this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); +_("The maximum output resolution of extracted video frames in pixels. Default: %(default)s"); +_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); +_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); +_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); _("Use this tag to build a DSM (Digital Surface Model, ground + objects) using a progressive morphological filter. Check the --dem* parameters for finer tuning. Default: %(default)s"); -_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); -_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); -_("The maximum vertex count of the output mesh. Default: %(default)s"); -_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); -_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); -_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); -_("Matcher algorithm, Fast Library for Approximate Nearest Neighbors or Bag of Words. FLANN is slower, but more stable. BOW is faster, but can sometimes miss valid matches. BRUTEFORCE is very slow but robust.Can be one of: %(choices)s. Default: %(default)s"); -_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); -_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); _("Use a full 3D mesh to compute the orthophoto instead of a 2.5D mesh. This option is a bit faster and provides similar results in planar areas. Default: %(default)s"); -_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); -_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); -_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); -_("The maximum number of processes to use in various processes. Peak memory requirement is ~1GB per thread and 2 megapixel image resolution. Default: %(default)s"); -_("Do not attempt to merge partial reconstructions. This can happen when images do not have sufficient overlap or are isolated. Default: %(default)s"); -_("Name of dataset (i.e subfolder name within project folder). Default: %(default)s"); -_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); -_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); -_("Set point cloud quality. Higher quality generates better, denser point clouds, but requires more memory and takes longer. Each step up in quality increases processing time roughly by a factor of 4x.Can be one of: %(choices)s. Default: %(default)s"); +_("Simple Morphological Filter elevation threshold parameter (meters). Default: %(default)s"); +_("DSM/DTM resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate. Default: %(default)s"); +_("When processing multispectral datasets, ODM will automatically align the images for each band. If the images have been postprocessed and are already aligned, use this option. Default: %(default)s"); +_("Set a value in meters for the GPS Dilution of Precision (DOP) information for all images. If your images are tagged with high precision GPS information (RTK), this value will be automatically set accordingly. You can use this option to manually set it in case the reconstruction fails. Lowering this option can sometimes help control bowling-effects over large areas. Default: %(default)s"); +_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); +_("Set this parameter if you want a striped GeoTIFF. Default: %(default)s"); +_("Skip generation of a full 3D model. This can save time if you only need 2D results such as orthophotos and DEMs. Default: %(default)s"); _("Keep faces in the mesh that are not seen in any camera. Default: %(default)s"); -_("Turn on rolling shutter correction. If the camera has a rolling shutter and the images were taken in motion, you can turn on this option to improve the accuracy of the results. See also --rolling-shutter-readout. Default: %(default)s"); -_("Skips dense reconstruction and 3D model generation. It generates an orthophoto directly from the sparse reconstruction. If you just need an orthophoto and do not need a full 3D model, turn on this option. Default: %(default)s"); +_("Create Cloud-Optimized GeoTIFFs instead of normal GeoTIFFs. Default: %(default)s"); +_("URL to a ClusterODM instance for distributing a split-merge workflow on multiple nodes in parallel. Default: %(default)s"); +_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Perform image matching with the nearest N images based on image filename order. Can speed up processing of sequential images, such as those extracted from video. It is applied only on non-georeferenced datasets. Set to 0 to disable. Default: %(default)s"); +_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); +_("Do not use GPU acceleration, even if it's available. Default: %(default)s"); +_("Permanently delete all previous results and rerun the processing pipeline."); _("Automatically crop image outputs by creating a smooth buffer around the dataset boundaries, shrunk by N meters. Use 0 to disable cropping. Default: %(default)s"); -_("Computes an euclidean raster map for each DEM. The map reports the distance from each cell to the nearest NODATA value (before any hole filling takes place). This can be useful to isolate the areas that have been filled. Default: %(default)s"); +_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); +_("Skip normalization of colors across all images. Useful when processing radiometric data. Default: %(default)s"); +_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); +_("Export the georeferenced point cloud in CSV format. Default: %(default)s"); +_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); +_("Displays version number and exits. "); +_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); _("Skip the blending of colors near seams. Default: %(default)s"); +_("Generates a polygon around the cropping area that cuts the orthophoto around the edges of features. This polygon can be useful for stitching seamless mosaics with multiple overlapping orthophotos. Default: %(default)s"); +_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); +_("Set the compression to use for orthophotos. Can be one of: %(choices)s. Default: %(default)s"); +_("Copy output results to this folder after processing."); +_("The maximum vertex count of the output mesh. Default: %(default)s"); _("Run local bundle adjustment for every image added to the reconstruction and a global adjustment every 100 images. Speeds up reconstruction for very large datasets. Default: %(default)s"); -_("Radius of the overlap between submodels. After grouping images into clusters, images that are closer than this radius to a cluster are added to the cluster. This is done to ensure that neighboring submodels overlap. Default: %(default)s"); -_("Path to the image geolocation file containing the camera center coordinates used for georeferencing. If you don't have values for yaw/pitch/roll you can set them to 0. The file needs to use the following format: EPSG: or <+proj definition>image_name geo_x geo_y geo_z [yaw (degrees)] [pitch (degrees)] [roll (degrees)] [horz accuracy (meters)] [vert accuracy (meters)]Default: %(default)s"); +_("Path to the project folder. Your project folder should contain subfolders for each dataset. Each dataset should have an \"images\" folder."); _("Choose the algorithm for extracting keypoints and computing descriptors. Can be one of: %(choices)s. Default: %(default)s"); -_("Generate OGC 3D Tiles outputs. Default: %(default)s"); -_("Rerun this stage only and stop. Can be one of: %(choices)s. Default: %(default)s"); -_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); -_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); -_("Use the camera parameters computed from another dataset instead of calculating them. Can be specified either as path to a cameras.json file or as a JSON string representing the contents of a cameras.json file. Default: %(default)s"); -_("Choose what to merge in the merge step in a split dataset. By default all available outputs are merged. Options: %(choices)s. Default: %(default)s"); -_("Delete heavy intermediate files to optimize disk space usage. This affects the ability to restart the pipeline from an intermediate stage, but allows datasets to be processed on machines that don't have sufficient disk space available. Default: %(default)s"); -_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Set this parameter if you want to generate a Google Earth (KMZ) rendering of the orthophoto. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the sky. Experimental. Default: %(default)s"); +_("show this help message and exit"); +_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Perform ground rectification on the point cloud. This means that wrongly classified ground points will be re-classified and gaps will be filled. Useful for generating DTMs. Default: %(default)s"); +_("Choose the structure from motion algorithm. For aerial datasets, if camera GPS positions and angles are available, triangulation can generate better results. For planar scenes captured at fixed altitude with nadir-only images, planar can be much faster. Can be one of: %(choices)s. Default: %(default)s"); +_("Number of steps used to fill areas with gaps. Set to 0 to disable gap filling. Starting with a radius equal to the output resolution, N different DEMs are generated with progressively bigger radius using the inverse distance weighted (IDW) algorithm and merged together. Remaining gaps are then merged using nearest neighbor interpolation. Default: %(default)s"); _("When processing multispectral datasets, you can specify the name of the primary band that will be used for reconstruction. It's recommended to choose a band which has sharp details and is in focus. Default: %(default)s"); -_("Maximum number of frames to extract from video files for processing. Set to 0 for no limit. Default: %(default)s"); -_("Path to the image groups file that controls how images should be split into groups. The file needs to use the following format: image_name group_nameDefault: %(default)s"); -_("Generate single file Binary glTF (GLB) textured models. Default: %(default)s"); +_("Set a camera projection type. Manually setting a value can help improve geometric undistortion. By default the application tries to determine a lens type from the images metadata. Can be one of: %(choices)s. Default: %(default)s"); +_("Path to a GeoTIFF DEM or a LAS/LAZ point cloud that the reconstruction outputs should be automatically aligned to. Experimental. Default: %(default)s"); _("Geometric estimates improve the accuracy of the point cloud by computing geometrically consistent depthmaps but may not be usable in larger datasets. This flag disables geometric estimates. Default: %(default)s"); -_("Permanently delete all previous results and rerun the processing pipeline."); -_("Override the rolling shutter readout time for your camera sensor (in milliseconds), instead of using the rolling shutter readout database. Note that not all cameras are present in the database. Set to 0 to use the database value. Default: %(default)s"); +_("Decimate the points before generating the DEM. 1 is no decimation (full quality). 100 decimates ~99%% of the points. Useful for speeding up generation of DEM results in very large datasets. Default: %(default)s"); +_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); +_("Automatically set a boundary using camera shot locations to limit the area of the reconstruction. This can help remove far away background artifacts (sky, background landscapes, etc.). See also --boundary. Default: %(default)s"); +_("Filters the point cloud by removing points that deviate more than N standard deviations from the local mean. Set to 0 to disable filtering. Default: %(default)s"); +_("Simple Morphological Filter elevation scalar parameter. Default: %(default)s"); +_("Export the georeferenced point cloud in Entwine Point Tile (EPT) format. Default: %(default)s"); +_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); _("Build orthophoto overviews for faster display in programs such as QGIS. Default: %(default)s"); -_("Set feature extraction quality. Higher quality generates better features, but requires more memory and takes longer. Can be one of: %(choices)s. Default: %(default)s"); -_("Generate static tiles for orthophotos and DEMs that are suitable for viewers like Leaflet or OpenLayers. Default: %(default)s"); -_("Displays version number and exits. "); -_("Minimum number of features to extract per image. More features can be useful for finding more matches between images, potentially allowing the reconstruction of areas with little overlap or insufficient features. More features also slow down processing. Default: %(default)s"); -_("Use images' GPS exif data for reconstruction, even if there are GCPs present.This flag is useful if you have high precision GPS measurements. If there are no GCPs, this flag does nothing. Default: %(default)s"); -_("Octree depth used in the mesh reconstruction, increase to get more vertices, recommended values are 8-12. Default: %(default)s"); -_("Rerun processing from this stage. Can be one of: %(choices)s. Default: %(default)s"); +_("Skip generation of PDF report. This can save time if you don't need a report. Default: %(default)s"); _("Classify the point cloud outputs. You can control the behavior of this option by tweaking the --dem-* parameters. Default: %(default)s"); -_("Export the georeferenced point cloud in LAS format. Default: %(default)s"); -_("Ignore Ground Sampling Distance (GSD).A memory and processor hungry change relative to the default behavior if set to true. Ordinarily, GSD estimates are used to cap the maximum resolution of image outputs and resizes images when necessary, resulting in faster processing and lower memory usage. Since GSD is an estimate, sometimes ignoring it can result in slightly better image output quality. Never set --ignore-gsd to true unless you are positive you need it, and even then: do not use it. Default: %(default)s"); +_("Orthophoto resolution in cm / pixel. Note that this value is capped by a ground sampling distance (GSD) estimate.Default: %(default)s"); +_("Simple Morphological Filter window radius parameter (meters). Default: %(default)s"); +_("GeoJSON polygon limiting the area of the reconstruction. Can be specified either as path to a GeoJSON file or as a JSON string representing the contents of a GeoJSON file. Default: %(default)s"); +_("Average number of images per submodel. When splitting a large dataset into smaller submodels, images are grouped into clusters. This value regulates the number of images that each cluster should have on average. Default: %(default)s"); +_("Use this tag if you have a GCP File but want to use the EXIF information for georeferencing instead. Default: %(default)s"); +_("Turn off camera parameter optimization during bundle adjustment. This can be sometimes useful for improving results that exhibit doming/bowling or when images are taken with a rolling shutter camera. Default: %(default)s"); _("Path to the file containing the ground control points used for georeferencing. The file needs to use the following format: EPSG: or <+proj definition>geo_x geo_y geo_z im_x im_y image_name [gcp_name] [extra1] [extra2]Default: %(default)s"); -_("Skip alignment of submodels in split-merge. Useful if GPS is good enough on very large datasets. Default: %(default)s"); +_("Set this parameter if you want to generate a PNG rendering of the orthophoto. Default: %(default)s"); +_("Automatically compute image masks using AI to remove the background. Experimental. Default: %(default)s"); +_("Generate OBJs that have a single material and a single texture file instead of multiple ones. Default: %(default)s"); +_("Generate OGC 3D Tiles outputs. Default: %(default)s"); +_("Save the georeferenced point cloud in Cloud Optimized Point Cloud (COPC) format. Default: %(default)s"); +_("Specify the distance between camera shot locations and the outer edge of the boundary when computing the boundary with --auto-boundary. Set to 0 to automatically choose a value. Default: %(default)s"); diff --git a/locale b/locale index 0a68f6ed5..64eee8cae 160000 --- a/locale +++ b/locale @@ -1 +1 @@ -Subproject commit 0a68f6ed5172a8838571a9872bcc3cb4f310794c +Subproject commit 64eee8cae41e5fe40a0123400f12600fcd6125b9 From 44b24952912fdda9a4c57c94544d154356f2c28a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 15:41:05 -0400 Subject: [PATCH 75/95] Fix raster export --- app/api/tiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/api/tiler.py b/app/api/tiler.py index 855eef569..8ae27bbd3 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -560,6 +560,9 @@ def post(self, request, pk=None, project_pk=None, asset_type=None): raise exceptions.ValidationError(_("Both formula and bands parameters are required")) if formula and bands: + if bands == 'auto': + bands, _ = get_auto_bands(task.orthophoto_bands, formula) + try: expr, _discard_ = lookup_formula(formula, bands) except ValueError as e: From 4897d4e52a911f0732ef29813f607785ec4aab42 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 3 Oct 2023 22:37:10 -0400 Subject: [PATCH 76/95] Fix var override --- app/api/tiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/tiler.py b/app/api/tiler.py index 8ae27bbd3..470930cdc 100644 --- a/app/api/tiler.py +++ b/app/api/tiler.py @@ -311,7 +311,7 @@ def get(self, request, pk=None, project_pk=None, tile_type="", z="", x="", y="", if hillshade == '' or hillshade == '0': hillshade = None if tilesize == '' or tilesize is None: tilesize = 256 if bands == 'auto' and formula: - bands, _ = get_auto_bands(task.orthophoto_bands, formula) + bands, _discard_ = get_auto_bands(task.orthophoto_bands, formula) try: tilesize = int(tilesize) @@ -561,7 +561,7 @@ def post(self, request, pk=None, project_pk=None, asset_type=None): if formula and bands: if bands == 'auto': - bands, _ = get_auto_bands(task.orthophoto_bands, formula) + bands, _discard_ = get_auto_bands(task.orthophoto_bands, formula) try: expr, _discard_ = lookup_formula(formula, bands) From 80dcff41ca39cf12c2ce1b4946ba0866822c2893 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:04:39 -0400 Subject: [PATCH 77/95] Add unit tests --- .../app/js/components/ProjectListItem.jsx | 2 -- app/tests/test_api_task.py | 16 +++++++++- app/tests/test_formulas.py | 32 +++++++++++++++++-- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 51023e933..31d6ec429 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -573,8 +573,6 @@ class ProjectListItem extends React.Component { render() { const { refreshing, data, filterTags } = this.state; const numTasks = data.tasks.length; - - // const numValidTasks = data.tasks.filter(t => ) const canEdit = this.hasPermission("change"); const userTags = Tags.userTags(data.tags); let deleteWarning = _("All tasks, images and models associated with this project will be permanently deleted. Are you sure you want to continue?"); diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index bf3af575a..a3df03470 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -497,6 +497,10 @@ def test_task(self): self.assertEqual(metadata['algorithms'], []) self.assertEqual(metadata['color_maps'], []) + # Auto bands + self.assertEqual(metadata['auto_bands']['filter'], '') + self.assertEqual(metadata['auto_bands']['match'], None) + # Address key is removed self.assertFalse('address' in metadata) @@ -531,6 +535,10 @@ def test_task(self): self.assertTrue(len(metadata['algorithms']) > 0) self.assertTrue(len(metadata['color_maps']) > 0) + # Auto band is populated + self.assertEqual(metadata['auto_bands']['filter'], 'RGN') + self.assertEqual(metadata['auto_bands']['match'], False) + # Algorithms have valid keys for k in ['id', 'filters', 'expr', 'help']: for a in metadata['algorithms']: @@ -557,6 +565,10 @@ def test_task(self): self.assertEqual(metadata['statistics']['1']['min'], algos['VARI']['range'][0]) self.assertEqual(metadata['statistics']['1']['max'], algos['VARI']['range'][1]) + # Formula can be set to auto + res = client.get("/api/projects/{}/tasks/{}/orthophoto/metadata?formula=VARI&bands=auto".format(project.id, task.id)) + self.assertEqual(res.status_code, status.HTTP_200_OK) + tile_path = { 'orthophoto': '17/32042/46185', 'dsm': '18/64083/92370', @@ -665,7 +677,9 @@ def test_task(self): ("orthophoto", "formula=VARI&bands=RGB", status.HTTP_200_OK), ("orthophoto", "formula=VARI&bands=invalid", status.HTTP_400_BAD_REQUEST), ("orthophoto", "formula=invalid&bands=RGB", status.HTTP_400_BAD_REQUEST), - + ("orthophoto", "formula=NDVI&bands=auto", status.HTTP_200_OK), + ("orthophoto", "formula=NDVI&bands=auto", status.HTTP_200_OK), + ("orthophoto", "formula=NDVI&bands=RGN&color_map=rdylgn&rescale=-1,1", status.HTTP_200_OK), ("orthophoto", "formula=NDVI&bands=RGN&color_map=rdylgn&rescale=1,-1", status.HTTP_200_OK), diff --git a/app/tests/test_formulas.py b/app/tests/test_formulas.py index ab9117da0..70fb9f382 100644 --- a/app/tests/test_formulas.py +++ b/app/tests/test_formulas.py @@ -1,6 +1,6 @@ import re from django.test import TestCase -from app.api.formulas import lookup_formula, get_algorithm_list, get_camera_filters_for, algos +from app.api.formulas import lookup_formula, get_algorithm_list, get_camera_filters_for, algos, get_auto_bands class TestFormulas(TestCase): def setUp(self): @@ -48,4 +48,32 @@ def test_algo_list(self): # Filters are less than 5 bands for f in i['filters']: bands = list(set(re.findall(pattern, f))) - self.assertTrue(len(bands) <= 5) \ No newline at end of file + self.assertTrue(len(bands) <= 5) + + def test_auto_bands(self): + obands = [{'name': 'red', 'description': 'red'}, + {'name': 'green', 'description': 'green'}, + {'name': 'blue', 'description': 'blue'}, + {'name': 'gray', 'description': 'nir'}, + {'name': 'alpha', 'description': None}] + + self.assertEqual(get_auto_bands(obands, "NDVI")[0], "RGBN") + self.assertTrue(get_auto_bands(obands, "NDVI")[1]) + + self.assertEqual(get_auto_bands(obands, "Celsius")[0], "L") + self.assertFalse(get_auto_bands(obands, "Celsius")[1]) + + self.assertEqual(get_auto_bands(obands, "VARI")[0], "RGBN") + self.assertTrue(get_auto_bands(obands, "VARI")[0]) + + obands = [{'name': 'red', 'description': None}, + {'name': 'green', 'description': None}, + {'name': 'blue', 'description': None}, + {'name': 'gray', 'description': None}, + {'name': 'alpha', 'description': None}] + + self.assertEqual(get_auto_bands(obands, "NDVI")[0], "RGN") + self.assertFalse(get_auto_bands(obands, "NDVI")[1]) + + self.assertEqual(get_auto_bands(obands, "VARI")[0], "RGB") + self.assertFalse(get_auto_bands(obands, "VARI")[1]) \ No newline at end of file From 13121566ad909fd466979448e3bbbfce8b045fbc Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:13:32 -0400 Subject: [PATCH 78/95] Update README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 976d2f145..0521f50c0 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,11 @@ A user-friendly, commercial grade software for drone image processing. Generate Windows and macOS users can purchase an automated [installer](https://www.opendronemap.org/webodm/download#installer), which makes the installation process easier. -To install WebODM manually, these steps should get you up and running: +There's also a cloud-hosted version of WebODM available from [webodm.net](https://webodm.net). -* Install the following applications (if they are not installed already): +To install WebODM manually on your machine: + +* Install the following applications: - [Git](https://git-scm.com/downloads) - [Docker](https://www.docker.com/) - [Docker-compose](https://docs.docker.com/compose/install/) From f67f435a1cac618a3b4a20c027a1c63b53c014b4 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:34:31 -0400 Subject: [PATCH 79/95] Ignore celery results when appropriate --- worker/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 392582b79..5ca8f24c2 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -31,7 +31,7 @@ # What class to use for async results, since during testing we need to mock it TestSafeAsyncResult = worker.celery.MockAsyncResult if settings.TESTING else app.AsyncResult -@app.task +@app.task(ignore_result=True) def update_nodes_info(): if settings.NODE_OPTIMISTIC_MODE: return @@ -57,7 +57,7 @@ def update_nodes_info(): processing_node.hostname = check_hostname processing_node.save() -@app.task +@app.task(ignore_result=True) def cleanup_projects(): # Delete all projects that are marked for deletion # and that have no tasks left @@ -68,7 +68,7 @@ def cleanup_projects(): logger.info("Deleted {} projects".format(count_dict['app.Project'])) -@app.task +@app.task(ignore_result=True) def cleanup_tmp_directory(): # Delete files and folder in the tmp directory that are # older than 24 hours @@ -99,7 +99,7 @@ def loop(): t.start() return stopped.set -@app.task +@app.task(ignore_result=True) def process_task(taskId): lock_id = 'task_lock_{}'.format(taskId) cancel_monitor = None @@ -159,7 +159,7 @@ def get_pending_tasks(): processing_node__isnull=False, partial=False) | Q(pending_action__isnull=False, partial=False)) -@app.task +@app.task(ignore_result=True) def process_pending_tasks(): tasks = get_pending_tasks() for task in tasks: @@ -207,7 +207,7 @@ def export_pointcloud(self, input, **opts): logger.error(str(e)) return {'error': str(e)} -@app.task +@app.task(ignore_result=True) def check_quotas(): profiles = Profile.objects.filter(quota__gt=-1) for p in profiles: From 62d5185a79a7857a6da17f26a4d565a962a993db Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 13:39:16 -0400 Subject: [PATCH 80/95] Fix test --- app/tests/test_api_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index a3df03470..d68d4504d 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -536,7 +536,7 @@ def test_task(self): self.assertTrue(len(metadata['color_maps']) > 0) # Auto band is populated - self.assertEqual(metadata['auto_bands']['filter'], 'RGN') + self.assertEqual(metadata['auto_bands']['filter'], '') self.assertEqual(metadata['auto_bands']['match'], False) # Algorithms have valid keys From 49c9f2d7b893a63339a12708f9f180db01011a55 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 4 Oct 2023 15:51:53 -0400 Subject: [PATCH 81/95] Fix test --- app/tests/test_api_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index d68d4504d..3c6d095ff 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -537,7 +537,7 @@ def test_task(self): # Auto band is populated self.assertEqual(metadata['auto_bands']['filter'], '') - self.assertEqual(metadata['auto_bands']['match'], False) + self.assertEqual(metadata['auto_bands']['match'], None) # Algorithms have valid keys for k in ['id', 'filters', 'expr', 'help']: From bc86c7977b6ba281af8378cccd4b9a6294c31040 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 5 Oct 2023 12:47:28 -0400 Subject: [PATCH 82/95] Add max number of pages in paginator --- app/static/app/js/components/Paginator.jsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/static/app/js/components/Paginator.jsx b/app/static/app/js/components/Paginator.jsx index 6944f74fa..cd3e19097 100644 --- a/app/static/app/js/components/Paginator.jsx +++ b/app/static/app/js/components/Paginator.jsx @@ -146,8 +146,16 @@ class Paginator extends React.Component { } if (itemsPerPage && itemsPerPage && totalItems > itemsPerPage){ - const numPages = Math.ceil(totalItems / itemsPerPage), - pages = [...Array(numPages).keys()]; // [0, 1, 2, ...numPages] + const numPages = Math.ceil(totalItems / itemsPerPage); + const MAX_PAGE_BUTTONS = 7; + + let rangeStart = Math.max(1, currentPage - Math.floor(MAX_PAGE_BUTTONS / 2)); + let rangeEnd = rangeStart + Math.min(numPages, MAX_PAGE_BUTTONS); + if (rangeEnd > numPages){ + rangeStart -= rangeEnd - numPages - 1; + rangeEnd -= rangeEnd - numPages - 1 + } + let pages = [...Array(rangeEnd - rangeStart).keys()].map(i => i + rangeStart - 1); paginator = (